diff options
Diffstat (limited to 'jjb')
47 files changed, 1720 insertions, 275 deletions
diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index da3992ea8..1c5f14880 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -62,11 +62,19 @@ auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' - 'os-odl-nofeature-ha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + - 'os-ovn-nofeature-ha': + auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-ovs-ha': + auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' # NOHA scenarios - 'os-nosdn-nofeature-noha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + exclude: + - scenario: os-ovn-nofeature-ha + stream: euphrates + jobs: - '{installer}-{scenario}-{pod}-daily-{stream}' - '{installer}-deploy-{pod}-daily-{stream}' @@ -122,15 +130,51 @@ DEPLOY_SCENARIO={scenario} same-node: true block: true + - trigger-builds: + - project: 'functest-{installer}-{pod}-arm-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - trigger-builds: + - project: 'yardstick-{installer}-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + # 1.here the stream means the SUT stream, dovetail stream is defined in its own job + # 2.testsuite proposed_tests here is for new test cases planning to add into OVP + # 3.run proposed_tests on Monday, Wednesday and Friday against ha scenario + # 4.testsuite default here is for the test cases already added into OVP + # 5.run default testsuite on Tuesday against ha scenario + # 6.not used for release criteria or compliance, + # only to debug the dovetail tool bugs with arm pods - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + MON: true + WED: true + FRI: true + use-build-time: true steps: - trigger-builds: - - project: 'functest-{installer}-{pod}-arm-daily-{stream}' + - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -141,14 +185,19 @@ failure-threshold: 'never' unstable-threshold: 'FAILURE' - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + TUES: true + use-build-time: true steps: - trigger-builds: - - project: 'yardstick-{installer}-{pod}-daily-{stream}' + - project: 'dovetail-{installer}-{pod}-default-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -158,21 +207,6 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.here the stream means the SUT stream, dovetail stream is defined in its own job - # 2.only debug testsuite here(refstack, ha, vping, ipv6, tempest, bgpvpn) - # 3.not used for release criteria or compliance, - # only to debug the dovetail tool bugs with arm pods - - trigger-builds: - - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' # Armband uses Fuel's log collection project job, no need to duplicate - conditional-step: condition-kind: not @@ -250,7 +284,7 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 2,4,6' + - timed: '0 1 * * 4,6' - trigger: name: 'fuel-os-nosdn-nofeature-noha-armband-baremetal-master-trigger' triggers: @@ -258,7 +292,15 @@ - trigger: name: 'fuel-os-odl-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 1,3,5,7' + - timed: '0 1 * * 2,5,7' +- trigger: + name: 'fuel-os-ovn-nofeature-ha-armband-baremetal-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-baremetal-master-trigger' + triggers: + - timed: '0 1 * * 1,3' # --------------------------------------------------------------------- # Enea Armband CI Baremetal Triggers running against euphrates branch # --------------------------------------------------------------------- @@ -274,6 +316,11 @@ name: 'fuel-os-odl-nofeature-ha-armband-baremetal-euphrates-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-baremetal-euphrates-trigger' + triggers: + - timed: '' + # -------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against master branch # -------------------------------------------------------------- @@ -289,6 +336,14 @@ name: 'fuel-os-odl-nofeature-ha-armband-virtual-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-ovn-nofeature-ha-armband-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-virtual-master-trigger' + triggers: + - timed: '' # ------------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against euphrates branch # ------------------------------------------------------------------- @@ -304,3 +359,7 @@ name: 'fuel-os-odl-nofeature-ha-armband-virtual-euphrates-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-virtual-euphrates-trigger' + triggers: + - timed: '' diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 81d76d559..357f24e37 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -123,6 +123,15 @@ - 'os-nosdn-bar-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-bar-noha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'k8-nosdn-stor4nfv-ha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'k8-nosdn-stor4nfv-noha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' jobs: - 'compass-{scenario}-{pod}-daily-{stream}' @@ -168,9 +177,7 @@ installer: '{installer}' gs-pathname: '{gs-pathname}' ppa-pathname: '{ppa-pathname}' - - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + scenario: '{scenario}' - '{slave-label}-defaults' - '{installer}-defaults' @@ -338,6 +345,7 @@ installer: '{installer}' gs-pathname: '{gs-pathname}' ppa-pathname: '{ppa-pathname}' + scenario: '{scenario}' - '{slave-label}-defaults' - '{installer}-defaults' @@ -384,6 +392,7 @@ installer: '{installer}' gs-pathname: '{gs-pathname}' ppa-pathname: '{ppa-pathname}' + scenario: '{scenario}' - '{slave-label}-defaults' - '{installer}-defaults' @@ -429,6 +438,17 @@ name: LOG_DIRECTORY default: $WORKSPACE/log_output description: "Directory where the logs will be located upon the completion of the collection." + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + description: "Scenario to deploy with." + - string: + name: COMPASS_OS_VERSION + default: '' + - string: + name: COMPASS_OPENSTACK_VERSION + default: '' + ######################## # trigger macros @@ -484,11 +504,15 @@ - trigger: name: 'compass-k8-nosdn-nofeature-ha-baremetal-centos-master-trigger' triggers: - - timed: '' + - timed: '0 6 * * *' - trigger: name: 'compass-os-nosdn-bar-ha-baremetal-centos-master-trigger' triggers: - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-centos-master-trigger' + triggers: + - timed: '' # ---------------------------- # noha-baremetal-centos-master @@ -517,6 +541,15 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-baremetal-centos-master-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-baremetal-centos-master-trigger' + triggers: + - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-baremetal-centos-master-trigger' + triggers: + - timed: '' + # -------------------------- # ha-huawei-pod7-danube @@ -573,6 +606,10 @@ name: 'compass-os-nosdn-bar-ha-huawei-pod7-danube-trigger' triggers: - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # ---------------------------- # noha-huawei-pod7-danube @@ -601,6 +638,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-huawei-pod7-danube-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # ------------------- # ha-baremetal-master @@ -648,7 +693,7 @@ - trigger: name: 'compass-k8-nosdn-nofeature-ha-baremetal-master-trigger' triggers: - - timed: '' + - timed: '0 10 2-30/2 * *' - trigger: name: 'compass-os-odl-sfc-ha-baremetal-master-trigger' triggers: @@ -657,6 +702,10 @@ name: 'compass-os-nosdn-bar-ha-baremetal-master-trigger' triggers: - timed: '0 2 2-30/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-master-trigger' + triggers: + - timed: '0 16 1-29/2 * *' # --------------------- # noha-baremetal-master @@ -685,6 +734,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-baremetal-master-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-baremetal-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-baremetal-master-trigger' + triggers: + - timed: '' # ------------------- # ha-baremetal-euphrates @@ -741,6 +798,10 @@ name: 'compass-os-nosdn-bar-ha-baremetal-euphrates-trigger' triggers: - timed: '0 21 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-euphrates-trigger' + triggers: + - timed: '' # --------------------- # noha-baremetal-euphrates @@ -769,6 +830,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-baremetal-euphrates-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-baremetal-euphrates-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-baremetal-euphrates-trigger' + triggers: + - timed: '' # ----------------- # ha-virtual-master @@ -825,6 +894,10 @@ name: 'compass-os-nosdn-bar-ha-virtual-master-trigger' triggers: - timed: '0 17 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-virtual-master-trigger' + triggers: + - timed: '0 15 2-30/2 * *' # ------------------- # noha-virtual-master @@ -853,6 +926,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-virtual-master-trigger' triggers: - timed: '0 11 2-30/2 * *' +- trigger: + name: 'compass-os-nosdn-bar-noha-virtual-master-trigger' + triggers: + - timed: '0 22 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-virtual-master-trigger' + triggers: + - timed: '0 10 2-30/2 * *' # ----------------- # ha-virtual-euphrates @@ -909,6 +990,10 @@ name: 'compass-os-nosdn-bar-ha-virtual-euphrates-trigger' triggers: - timed: '0 19 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-virtual-euphrates-trigger' + triggers: + - timed: '' # '0 14 1-29/2 * *' # ------------------- # noha-virtual-euphrates @@ -937,3 +1022,11 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-virtual-euphrates-trigger' triggers: - timed: '0 12 2-30/2 * *' +- trigger: + name: 'compass-os-nosdn-bar-noha-virtual-euphrates-trigger' + triggers: + - timed: '0 12 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-virtual-euphrates-trigger' + triggers: + - timed: '' # '0 12 2-30/2 * *' diff --git a/jjb/container4nfv/arm64/compass-deploy.sh b/jjb/container4nfv/arm64/compass-deploy.sh index bbbf3f449..a22af09ba 100755 --- a/jjb/container4nfv/arm64/compass-deploy.sh +++ b/jjb/container4nfv/arm64/compass-deploy.sh @@ -5,9 +5,14 @@ cd compass4nfv export ADAPTER_OS_PATTERN='(?i)CentOS-7.*arm.*' export OS_VERSION="centos7" -export KUBERNETES_VERSION="v1.7.3" -export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml" -export NETWORK="deploy/conf/vm_environment/network.yml" -export VIRT_NUMBER=2 VIRT_CPUS=4 VIRT_MEM=8192 VIRT_DISK=50G +export KUBERNETES_VERSION="v1.7.5" +if [[ "$NODE_NAME" =~ "-virtual" ]]; then + export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml" + export NETWORK="deploy/conf/vm_environment/network.yml" + export VIRT_NUMBER=2 VIRT_CPUS=4 VIRT_MEM=8192 VIRT_DISK=50G +else + export DHA="deploy/conf/hardware_environment/huawei-pod8/k8-nosdn-nofeature-noha.yml" + export NETWORK="deploy/conf/hardware_environment/huawei-pod8/network.yml" +fi ./deploy.sh diff --git a/jjb/container4nfv/container4nfv-arm64.yml b/jjb/container4nfv/container4nfv-arm64.yml index b69c21400..324624387 100644 --- a/jjb/container4nfv/container4nfv-arm64.yml +++ b/jjb/container4nfv/container4nfv-arm64.yml @@ -9,15 +9,20 @@ disabled: false - 'k8-sriov-nofeature-noha': disabled: false + pod: + - virtual: + slave-label: arm-packet01 + - baremetal: + slave-label: compass-baremetal-arm jobs: - - 'container4nfv-{scenario}-virtual-daily-master' + - 'container4nfv-{scenario}-{pod}-daily-master' - job-template: - name: 'container4nfv-{scenario}-virtual-daily-master' + name: 'container4nfv-{scenario}-{pod}-daily-master' disabled: '{obj:disabled}' concurrent: false - node: arm-packet01 + node: '{slave-label}' scm: - git: @@ -28,7 +33,7 @@ wipe-workspace: true triggers: - - 'trigger-{scenario}-virtual' + - 'trigger-{scenario}-{pod}' wrappers: - timeout: @@ -46,6 +51,7 @@ same-node: true block: false + - trigger: name: 'trigger-k8-multus-nofeature-noha-virtual' triggers: @@ -54,3 +60,12 @@ name: 'trigger-k8-sriov-nofeature-noha-virtual' triggers: - timed: '0 16 * * *' + +- trigger: + name: 'trigger-k8-multus-nofeature-noha-baremetal' + triggers: + - timed: '0 16 * * *' +- trigger: + name: 'trigger-k8-sriov-nofeature-noha-baremetal' + triggers: + - timed: '0 20 * * *' diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index fbbb086d7..043e40fb8 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -117,9 +117,7 @@ - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' - - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + - 'testapi-parameter' - 'daisy-project-parameter': gs-pathname: '{gs-pathname}' @@ -192,8 +190,10 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'daisy.*-deploy-({pod})?-daily-.*' - 'daisy-kolla-build-.*' + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' block-level: 'NODE' parameters: @@ -203,9 +203,7 @@ - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' - - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' + - 'testapi-parameter' - 'daisy-project-parameter': gs-pathname: '{gs-pathname}' - string: diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index 006153f6f..c4edd7738 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -215,6 +215,13 @@ enabled: true max-total: 6 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + block-level: 'NODE' scm: - git-scm diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index ba76b8abb..8c04ffd53 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -19,7 +19,7 @@ installer: - 'apex' - 'fuel' - # - 'joid' + - 'daisy' arch: - 'x86_64' @@ -29,19 +29,17 @@ - 'sample' - 'congress' - task: - - verify: - auto-trigger-name: 'doctor-verify' - exclude: - installer: 'apex' arch: 'aarch64' + - installer: 'daisy' + arch: 'aarch64' jobs: - 'doctor-verify-{stream}' - 'doctor-verify-unit-test-{stream}' - - 'doctor-{task}-{inspector}-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + - 'doctor-verify-{inspector}-{stream}' + - 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' - job-template: name: 'doctor-verify-{stream}' @@ -76,6 +74,7 @@ file-paths: - compare-type: ANT pattern: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -117,16 +116,17 @@ - 'doctor-verify-unit-test-publishers-macro' - job-template: - name: 'doctor-{task}-{inspector}-{stream}' + name: 'doctor-verify-{inspector}-{stream}' disabled: '{obj:disabled}' project-type: 'multijob' scm: - git-scm-gerrit triggers: - - '{auto-trigger-name}': + - 'doctor-verify': project: '{project}' branch: '{branch}' files: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -137,7 +137,7 @@ name: 'doctor-verify-apex-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-apex-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-apex-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -151,7 +151,7 @@ name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -165,7 +165,20 @@ name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-aarch64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-aarch64-{stream}' + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + kill-phase-on: FAILURE + current-parameters: true + git-revision: true + - multijob: + name: 'doctor-verify-daisy-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-verify-daisy-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -177,7 +190,7 @@ git-revision: true - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + name: 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' disabled: '{obj:disabled}' node: 'doctor-{installer}-{arch}' wrappers: diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 451662abe..ec879e325 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -199,6 +199,27 @@ if [[ ! -f ${cirros_image} ]]; then fi sudo cp ${cirros_image} ${DOVETAIL_CONFIG} +# snaps_smoke test case needs to download this image first before running +ubuntu14_image=${image_path}/ubuntu-14.04-server-cloudimg-amd64-disk1.img +if [[ ! -f ${ubuntu14_image} ]]; then + echo "Download image ubuntu-14.04-server-cloudimg-amd64-disk1.img ..." + wget -q -nc https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img -P ${image_path} +fi +sudo cp ${ubuntu14_image} ${DOVETAIL_CONFIG} + +# cloudify_ims test case needs to download these 2 images first before running +cloudify_image=${image_path}/cloudify-manager-premium-4.0.1.qcow2 +if [[ ! -f ${cloudify_image} ]]; then + echo "Download image cloudify-manager-premium-4.0.1.qcow2 ..." + wget -q -nc http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 -P ${image_path} +fi +sudo cp ${cloudify_image} ${DOVETAIL_CONFIG} +trusty_image=${image_path}/trusty-server-cloudimg-amd64-disk1.img +if [[ ! -f ${trusty_image} ]]; then + echo "Download image trusty-server-cloudimg-amd64-disk1.img ..." + wget -q -nc http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img -P ${image_path} +fi +sudo cp ${trusty_image} ${DOVETAIL_CONFIG} opts="--privileged=true -id" @@ -238,18 +259,6 @@ if [ $(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | wc -l) == 0 ]; then exit 1 fi -if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then - source_cmd="source ${OPENRC}" - get_public_url_cmd="openstack --insecure endpoint list --service keystone --interface public | sed -n 4p | awk '{print \$14}'" - public_url=$(sudo docker exec "$container_id" /bin/bash -c "${source_cmd} && ${get_public_url_cmd}") - sed -i 's#OS_AUTH_URL=.*#OS_AUTH_URL='"${public_url}"'#g' ${OPENRC} - sed -i 's/internal/public/g' ${OPENRC} - if [[ ${public_url} =~ 'v2' ]]; then - sed -i "s/OS_IDENTITY_API_VERSION=3/OS_IDENTITY_API_VERSION=2.0/g" ${OPENRC} - fi - cat ${OPENRC} -fi - # Modify tempest_conf.yaml file tempest_conf_file=${DOVETAIL_CONFIG}/tempest_conf.yaml if [[ ${INSTALLER_TYPE} == 'compass' || ${INSTALLER_TYPE} == 'apex' ]]; then diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index bbc8a8493..a198751a8 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -63,6 +63,8 @@ auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-nofeature-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' + - 'os-ovn-nofeature-ha': + auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-ovs-ha': @@ -101,6 +103,14 @@ stream: master - scenario: os-odl_l2-bgpvpn-ha stream: euphrates + - scenario: os-ovn-nofeature-noha + stream: danube + - scenario: os-ovn-nofeature-noha + stream: euphrates + - scenario: os-ovn-nofeature-ha + stream: danube + - scenario: os-ovn-nofeature-ha + stream: euphrates jobs: - 'fuel-{scenario}-{pod}-daily-{stream}' @@ -161,33 +171,73 @@ DEPLOY_SCENARIO={scenario} same-node: true block: true + - trigger-builds: + - project: 'functest-fuel-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - trigger-builds: + - project: 'yardstick-fuel-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + # 1.here the stream means the SUT stream, dovetail stream is defined in its own job + # 2.testsuite proposed_tests here is for new test cases planning to add into OVP + # 3.run proposed_tests on Monday, Wednesday and Friday against ha scenario + # 4.testsuite default here is for the test cases already added into OVP + # 5.run default testsuite on Tuesday against ha scenario + # 6.not used for release criteria or compliance, only to debug the dovetail tool bugs - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + MON: true + WED: true + FRI: true + use-build-time: true steps: - trigger-builds: - - project: 'functest-fuel-{pod}-daily-{stream}' + - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} - same-node: true block: true + same-node: true block-thresholds: build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + TUES: true + use-build-time: true steps: - trigger-builds: - - project: 'yardstick-fuel-{pod}-daily-{stream}' + - project: 'dovetail-fuel-{pod}-default-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -197,21 +247,6 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.here the stream means the SUT stream, dovetail stream is defined in its own job - # 2.only debug testsuite here(refstack, ha, vping, ipv6, tempest, bgpvpn) - # 3.not used for release criteria or compliance, only to debug the dovetail tool bugs - # 4.ZTE pod1, os-nosdn-nofeature-ha and os-odl-bgpvpn-ha, run against danube - - trigger-builds: - - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - conditional-step: condition-kind: not condition-operand: @@ -357,6 +392,10 @@ triggers: - timed: '' # '5 8 * * *' - trigger: + name: 'fuel-os-ovn-nofeature-ha-baremetal-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-ha-baremetal-daily-master-trigger' triggers: - timed: '' # '5 17 * * *' @@ -459,10 +498,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-ovn-nofeature-noha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-baremetal-daily-euphrates-trigger' triggers: - timed: '' @@ -498,6 +533,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-ha-virtual-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-ha-virtual-daily-master-trigger' triggers: - timed: '' @@ -603,10 +642,6 @@ triggers: - timed: '' # '0 23 * * *' - trigger: - name: 'fuel-os-ovn-nofeature-noha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-euphrates-trigger' triggers: - timed: '' # '30 6 * * *' @@ -642,6 +677,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-ha-zte-pod1-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod1-daily-master-trigger' triggers: - timed: '' diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index 099d01a59..a58e0bf3f 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -197,9 +197,9 @@ - 'suite': job-timeout: 60 - 'daily': - job-timeout: 300 + job-timeout: 360 - 'arm-daily': - job-timeout: 300 + job-timeout: 360 jobs: - 'functest-{installer}-{pod}-{testsuite}-{stream}' diff --git a/jjb/releng/functest-docker.yml b/jjb/functest/functest-docker.yml index 584dc302c..909c7b079 100644 --- a/jjb/releng/functest-docker.yml +++ b/jjb/functest/functest-docker.yml @@ -72,7 +72,7 @@ disabled: '{obj:disabled}' parameters: - - job-parameters: + - functest-job-parameters: project: '{project}' branch: '{branch}' slave_label: 'opnfv-build-ubuntu' @@ -168,7 +168,7 @@ name: 'functest-{image}-docker-build-{arch_tag}-{stream}' disabled: '{obj:disabled}' parameters: - - job-parameters: + - functest-job-parameters: project: '{project}' branch: '{branch}' slave_label: '{slave_label}' @@ -205,6 +205,11 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + disabled: '{obj:disabled}' @@ -219,13 +224,13 @@ esac sudo manifest-tool push from-args \ --platforms linux/amd64,linux/arm64 \ - --template opnfv/functest-{image}:ARCH-$tag \ - --target opnfv/functest-{image}:$tag + --template $REPO/functest-{image}:ARCH-$tag \ + --target $REPO/functest-{image}:$tag exit $? # parameter macro - parameter: - name: job-parameters + name: functest-job-parameters parameters: - project-parameter: project: '{project}' @@ -262,6 +267,10 @@ name: PROJECT default: "{project}" description: "Project name used to enable job conditions" + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" # publisher macros - publisher: @@ -281,3 +290,4 @@ jalausuch@suse.com morgan.richomme@orange.com cedric.ollivier@orange.com feng.xiaowei@zte.com.cn juha.kosonen@nokia.com wangwulin@huawei.com + valentin.boucher@kontron.com diff --git a/jjb/functest/functest-k8.sh b/jjb/functest/functest-k8.sh index 888aded81..3f08e4a0b 100755 --- a/jjb/functest/functest-k8.sh +++ b/jjb/functest/functest-k8.sh @@ -18,6 +18,9 @@ if [[ ${INSTALLER_TYPE} == 'compass' ]]; then echo "export KUBE_MASTER_URL=$KUBE_MASTER_URL" >> $rc_file KUBE_MASTER_IP=$(echo $KUBE_MASTER_URL|awk -F'https://|:[0-9]+' '$0=$2') echo "export KUBE_MASTER_IP=$KUBE_MASTER_IP" >> $rc_file +elif [[ ${INSTALLER_TYPE} == 'joid' && ${BRANCH} == 'master' ]]; then + admin_conf_file_vol="-v ${HOME}/joid_config/config:/root/joid_config/config" + rc_file=${HOME}/joid_config/k8config else echo "Not supported by other installers yet" exit 1 @@ -44,9 +47,10 @@ envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} \ DOCKER_TAG=`[[ ${BRANCH##*/} == "master" ]] && echo "latest" || echo ${BRANCH##*/}` FUNCTEST_IMAGE=opnfv/functest-kubernetes:${DOCKER_TAG} +echo "Pulling Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null cmd_opt="run_tests -r -t all" -cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" +cmd="docker run --rm --privileged=true ${envs} ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest k8s test cases, CMD: ${cmd}" eval ${cmd} ret_value=$? diff --git a/jjb/functest/functest-kubernetes-project-jobs.yml b/jjb/functest/functest-kubernetes-project-jobs.yml new file mode 100644 index 000000000..3eba5d177 --- /dev/null +++ b/jjb/functest/functest-kubernetes-project-jobs.yml @@ -0,0 +1,257 @@ +--- +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: functest-kubernetes-project-jobs + + project: 'functest-kubernetes' + + jobs: + - 'functest-kubernetes-verify-{stream}' + - 'functest-kubernetes-verify-{phase}-{stream}' + - 'functest-kubernetes-docs-upload-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - euphrates: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false + + phase: + - 'unit-tests-and-docs': + slave-label: 'opnfv-build-ubuntu' + - 'build-x86_64': + slave-label: 'opnfv-build-ubuntu' + - 'build-aarch64': + slave-label: 'opnfv-build-ubuntu-arm' + +- job-template: + name: 'functest-kubernetes-verify-{stream}' + + disabled: '{obj:disabled}' + + project-type: 'multijob' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - 'functest-kubernetes-verify-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'functest-kubernetes-build-and-unittest' + execution-type: PARALLEL + projects: + - name: 'functest-kubernetes-verify-unit-tests-and-docs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'functest-kubernetes-verify-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=x86_64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'functest-kubernetes-verify-build-aarch64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=aarch64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + +- job-template: + name: 'functest-kubernetes-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 30 + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{slave-label}-defaults' + + scm: + - git-scm-gerrit + + builders: + - 'functest-kubernetes-verify-{phase}-builders-macro' + + publishers: + - 'functest-kubernetes-verify-{phase}-publishers-macro' + +- job-template: + name: 'functest-kubernetes-docs-upload-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - 'functest-kubernetes-docs-upload-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - functest-kubernetes-upload-doc-artifact + +################################ +# job triggers +################################ +- trigger: + name: 'functest-kubernetes-verify-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + +- trigger: + name: 'functest-kubernetes-docs-upload-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' +################################ +# job builders +################################ + +- builder: + name: 'functest-kubernetes-verify-unit-tests-and-docs-builders-macro' + builders: + - shell: | + cd $WORKSPACE && tox + +- builder: + name: 'functest-kubernetes-verify-build-x86_64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'functest-kubernetes-verify-build-aarch64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'functest-kubernetes-upload-doc-artifact' + builders: + - shell: | + cd $WORKSPACE && tox -edocs + wget -O - https://git.opnfv.org/releng/plain/utils/upload-artifact.sh | bash -s "api/_build" "docs" +################################ +# job publishers +################################ +- publisher: + name: 'functest-kubernetes-verify-unit-tests-and-docs-publishers-macro' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + - email-jenkins-admins-on-failure + +- publisher: + name: 'functest-kubernetes-verify-build-x86_64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure + +- publisher: + name: 'functest-kubernetes-verify-build-aarch64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure diff --git a/jjb/functest/xtesting-docker.yml b/jjb/functest/xtesting-docker.yml new file mode 100644 index 000000000..dc26344e3 --- /dev/null +++ b/jjb/functest/xtesting-docker.yml @@ -0,0 +1,221 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: xtesting-docker + + project: functest-xtesting + + stream: + - master: + branch: '{stream}' + disabled: false + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + - 'arm64': + slave_label: 'opnfv-build-ubuntu-arm' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + GERRIT_REFNAME=$GERRIT_REFNAME + DOCKERFILE=$DOCKERFILE + kill-phase-on: FAILURE + abort-all-jobs: false + + manifest-job-settings: &manifest-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: + GERRIT_REFNAME=$GERRIT_REFNAME + kill-phase-on: FAILURE + abort-all-jobs: false + + # yamllint enable rule:key-duplicates + jobs: + - "xtesting-docker-{stream}" + - "xtesting-docker-build-{arch_tag}-{stream}" + - "xtesting-docker-manifest-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'xtesting-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - xtesting-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/30 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' + + builders: + - multijob: + name: 'build xtesting images' + execution-type: PARALLEL + projects: + - name: 'xtesting-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'xtesting-docker-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish xtesting manifests' + execution-type: PARALLEL + projects: + - name: 'xtesting-docker-manifest-{stream}' + <<: *manifest-job-settings + + publishers: + - 'xtesting-amd64-recipients' + - 'xtesting-arm64-recipients' + +- job-template: + name: 'xtesting-docker-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - xtesting-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + scm: + - git-scm + builders: + - shell: | + #!/bin/bash -ex + sudo arch={arch_tag} bash ./build.sh + exit $? + +- job-template: + name: 'xtesting-docker-manifest-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'opnfv-build-ubuntu' + description: 'Slave label on Jenkins' + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + + + disabled: '{obj:disabled}' + + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template $REPO/xtesting:ARCH-$tag \ + --target $REPO/xtesting:$tag + exit $? + +- parameter: + name: xtesting-job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: GERRIT_REFNAME + default: "" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" + - string: + name: DOCKERFILE + default: "Dockerfile" + description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "{arch_tag}" + description: "If set, this value will be added to the docker image tag as a prefix" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + +# publisher macros +- publisher: + name: 'xtesting-arm64-recipients' + publishers: + - email: + recipients: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + delia.popescu@enea.com + +- publisher: + name: 'xtesting-amd64-recipients' + publishers: + - email: + recipients: > + jalausuch@suse.com morgan.richomme@orange.com + cedric.ollivier@orange.com feng.xiaowei@zte.com.cn + juha.kosonen@nokia.com wangwulin@huawei.com + valentin.boucher@kontron.com diff --git a/jjb/functest/xtesting-project-jobs.yml b/jjb/functest/xtesting-project-jobs.yml new file mode 100644 index 000000000..bf08775b6 --- /dev/null +++ b/jjb/functest/xtesting-project-jobs.yml @@ -0,0 +1,257 @@ +--- +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: xtesting-project-jobs + + project: 'functest-xtesting' + + jobs: + - 'xtesting-verify-{stream}' + - 'xtesting-verify-{phase}-{stream}' + - 'xtesting-docs-upload-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - euphrates: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false + + phase: + - 'unit-tests-and-docs': + slave-label: 'opnfv-build-ubuntu' + - 'build-x86_64': + slave-label: 'opnfv-build-ubuntu' + - 'build-aarch64': + slave-label: 'opnfv-build-ubuntu-arm' + +- job-template: + name: 'xtesting-verify-{stream}' + + disabled: '{obj:disabled}' + + project-type: 'multijob' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - 'xtesting-verify-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'xtesting-build-and-unittest' + execution-type: PARALLEL + projects: + - name: 'xtesting-verify-unit-tests-and-docs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'xtesting-verify-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=x86_64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'xtesting-verify-build-aarch64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=aarch64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + +- job-template: + name: 'xtesting-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 30 + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{slave-label}-defaults' + + scm: + - git-scm-gerrit + + builders: + - 'xtesting-verify-{phase}-builders-macro' + + publishers: + - 'xtesting-verify-{phase}-publishers-macro' + +- job-template: + name: 'xtesting-docs-upload-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - 'xtesting-docs-upload-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - xtesting-upload-doc-artifact + +################################ +# job triggers +################################ +- trigger: + name: 'xtesting-verify-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + +- trigger: + name: 'xtesting-docs-upload-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' +################################ +# job builders +################################ + +- builder: + name: 'xtesting-verify-unit-tests-and-docs-builders-macro' + builders: + - shell: | + cd $WORKSPACE && tox + +- builder: + name: 'xtesting-verify-build-x86_64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'xtesting-verify-build-aarch64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'xtesting-upload-doc-artifact' + builders: + - shell: | + cd $WORKSPACE && tox -edocs + wget -O - https://git.opnfv.org/releng/plain/utils/upload-artifact.sh | bash -s "api/_build" "docs" +################################ +# job publishers +################################ +- publisher: + name: 'xtesting-verify-unit-tests-and-docs-publishers-macro' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + - email-jenkins-admins-on-failure + +- publisher: + name: 'xtesting-verify-build-x86_64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure + +- publisher: + name: 'xtesting-verify-build-aarch64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure diff --git a/jjb/global/installer-report.sh b/jjb/global/installer-report.sh index e2fcfd6f6..6cd83f1f1 100755 --- a/jjb/global/installer-report.sh +++ b/jjb/global/installer-report.sh @@ -8,18 +8,39 @@ ############################################################################## source $WORKSPACE/installer_track.sh -echo """ - INSTALLER: $INSTALLER - INSTALLER_VERSION: $INSTALLER_VERSION - JOB_NAME: $JOB_NAME - BUILD_ID: $BUILD_ID - SENARIO: $DEPLOY_SCENARIO - UPSTREAM_JOB_NAME: $UPSTREAM_JOB_NAME: - UPSTREAM_BUILD_ID: $UPSTREAM_BUILD_ID - PROVISION_RESULT: $PROVISION_RESULT - TIMESTAMP_START: $TIMESTAMP_START - TIMESTAMP_END: `date '+%Y-%m-%d %H:%M:%S.%3N'` - POD_NAME: $NODE_NAME -""" -# TODO call TestAPI to report installer provisoin result when API is ready +gen_content() +{ + cat <<EOF +{ + "installer": "$INSTALLER", + "version": "$INSTALLER_VERSION", + "pod_name": "$NODE_NAME", + "job_name": "$JOB_NAME", + "build_id": "$BUILD_ID", + "scenario": "$DEPLOY_SCENARIO", + "upstream_job_name": "$UPSTREAM_JOB_NAME", + "upstream_build_id":"$UPSTREAM_BUILD_ID", + "criteria": "$PROVISION_RESULT", + "start_date": "$TIMESTAMP_START", + "stop_date": "$TIMESTAMP_END", + "details":"" +} +EOF +} + +echo "Installer: $INSTALLER provision result: $PROVISION_RESULT" +echo $(gen_content) + +set -o xtrace +curl -H "Content-Type: application/json" -X POST -v -d "$(gen_content)" \ + $TESTAPI_URL/deployresults || true + +# INFO +# postbuildscript plugin shall always return the original job running status, +# for the result returned from postbuildscript affects the CI pipeline. +if [ "$PROVISION_RESULT" == "PASS" ]; then + exit 0 +else + exit 1 +fi diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index a12a3c8a5..205802fab 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -40,6 +40,30 @@ default: 'refs/heads/{branch}' description: "Default refspec needed for manually triggering." +- parameter: + name: testapi-parameter + parameters: + - string: + name: TESTAPI_URL + default: 'http://testresults.opnfv.org/test/api/v1' + description: "Default TestAPI URL, currently using v1" + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + description: "OPNFV deployment scenario" + - string: + name: INSTALLER_VERSION + default: 'master' + description: "Installer release version" + - string: + name: UPSTREAM_JOB_NAME + default: '' + description: "Parent job name in Jenkins" + - string: + name: UPSTREAM_BUILD_ID + default: '' + description: "Parent job build_id in Jenkins" + - property: name: logrotate-default properties: @@ -318,7 +342,9 @@ - shell: | #!/bin/bash if [[ -s violation.log ]]; then - echo "Reporting lint result..." + cat violation.log + echo "Reporting lint result...." + set -x msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset." sed -i -e "1s#^#${msg}\n\n#" violation.log cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE" @@ -901,7 +927,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - ManuelBuilmbuil@suse.com + mbuil@suse.com - publisher: name: 'email-snaps-ptl' @@ -958,15 +984,18 @@ script-only-if-succeeded: true builders: - shell: | - echo "export PROVISION_RESULT=SUCCEED" >> $WORKSPACE/installer_track.sh - echo "export INSTALLER={installer}" >> $WORKSPACE/installer_track.sh + echo "export PROVISION_RESULT=PASS" >> $WORKSPACE/installer_track.sh + echo "export INSTALLER=$INSTALLER_TYPE" >> $WORKSPACE/installer_track.sh + echo "export TIMESTAMP_END="\'`date '+%Y-%m-%d %H:%M:%S.%3N'`\' >> $WORKSPACE/installer_track.sh - shell: - !include-raw-escape: installer-report.sh + !include-raw: installer-report.sh - postbuildscript: + script-only-if-succeeded: false script-only-if-failed: true builders: - shell: | echo "export PROVISION_RESULT=FAIL" >> $WORKSPACE/installer_track.sh - echo "export INSTALLER={installer}" >> $WORKSPACE/installer_track.sh + echo "export INSTALLER=$INSTALLER_TYPE" >> $WORKSPACE/installer_track.sh + echo "export TIMESTAMP_END="\'`date '+%Y-%m-%d %H:%M:%S.%3N'`\' >> $WORKSPACE/installer_track.sh - shell: - !include-raw-escape: installer-report.sh + !include-raw: installer-report.sh diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 86b369bcd..6171a6abd 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -983,6 +983,7 @@ - 'doctor-fuel-x86_64' - 'doctor-fuel-aarch64' - 'doctor-joid-x86_64' + - 'doctor-daisy-x86_64' - 'not-specified' default-slaves: - 'not-specified' diff --git a/jjb/pharos/check-jinja2.yml b/jjb/pharos/check-jinja2.yml index 7251d3f61..3ef2f04e5 100644 --- a/jjb/pharos/check-jinja2.yml +++ b/jjb/pharos/check-jinja2.yml @@ -68,11 +68,11 @@ - compare-type: ANT pattern: '**/*.yaml' - compare-type: ANT - pattern: 'config/installers/**/*.j2' + pattern: 'config/utils/*.py' - compare-type: ANT - pattern: 'check-jinja2.sh' + pattern: 'config/installers/**/*.j2' - compare-type: ANT - pattern: 'check-schema.sh' + pattern: 'check-*.sh' skip-vote: successful: false failed: false diff --git a/jjb/pharos/pharos.yml b/jjb/pharos/pharos.yml index acf14885e..c1bb1ba0e 100644 --- a/jjb/pharos/pharos.yml +++ b/jjb/pharos/pharos.yml @@ -11,7 +11,8 @@ jobs: - '{project}-verify-basic' - 'backup-pharos-dashboard' - - 'deploy-pharos-dashboard' + - 'deploy-pharos-dashboard': + disabled: true - job-template: name: 'backup-pharos-dashboard' @@ -52,7 +53,16 @@ - 'pharos-dashboard-defaults' scm: - - git-scm-gerrit + - git: + choosing-strategy: 'gerrit' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$BRANCH' + timeout: 15 + credentials-id: '$SSH_CREDENTIAL_ID' + url: '$GIT_BASE' + skip-tag: true + wipe-workspace: false triggers: - gerrit: @@ -71,6 +81,8 @@ builders: - shell: | cp $HOME/config.env $WORKSPACE/dashboard + cp $HOME/rsa.pub $WORKSPACE/dashboard + cp $HOME/rsa.pem $WORKSPACE/dashboard cd $WORKSPACE/dashboard sudo docker-compose build sudo docker-compose up -d diff --git a/jjb/qtip/qtip-experimental-jobs.yml b/jjb/qtip/qtip-experimental-jobs.yml index 103069fb7..b130912f3 100644 --- a/jjb/qtip/qtip-experimental-jobs.yml +++ b/jjb/qtip/qtip-experimental-jobs.yml @@ -13,10 +13,6 @@ branch: '{stream}' gs-pathname: '' disabled: false - - euphrates: - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false pod: - zte-virtual6: diff --git a/jjb/qtip/qtip-validate-jobs.yml b/jjb/qtip/qtip-validate-jobs.yml index d9d3e72e6..5c0bb881b 100644 --- a/jjb/qtip/qtip-validate-jobs.yml +++ b/jjb/qtip/qtip-validate-jobs.yml @@ -27,22 +27,17 @@ - compute: installer: apex pod: zte-virtual5 + sut: node <<: *master - storage: installer: apex pod: zte-virtual5 + sut: '' <<: *master - compute: - installer: apex - pod: zte-virtual5 - <<: *euphrates - - storage: - installer: apex - pod: zte-virtual5 - <<: *euphrates - - compute: installer: fuel pod: zte-virtual6 + sut: vnf <<: *master # ------------------------------- @@ -84,6 +79,10 @@ - string: name: OPNFV_RELEASE default: '{stream}' + - string: + name: SUT + description: "System Under Test" + default: '{sut}' scm: - git-scm diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml index 7f142241c..5fbb37ca2 100644 --- a/jjb/releng/compass4nfv-docker.yml +++ b/jjb/releng/compass4nfv-docker.yml @@ -27,6 +27,8 @@ - 'db' - 'deck' - 'tasks-base' + - 'tasks-k8s' + - 'tasks-osa' # settings for jobs run in multijob phases build-job-settings: &build-job-settings @@ -95,6 +97,10 @@ <<: *build-job-settings - name: 'compass-tasks-build-amd64-{stream}' <<: *build-job-settings + - name: 'compass-tasks-k8s-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-tasks-osa-build-amd64-{stream}' + <<: *build-job-settings publishers: - 'compass-amd64-recipients' diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index 28e5b7ce1..5c8cafbe3 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -111,7 +111,7 @@ fi echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" echo "--------------------------------------------------------" echo -cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH +cmd="docker build --pull=true --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH $ARCH_BUILD_ARG -f $DOCKERFILE $DOCKER_PATH" diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 317efc807..55cbc5f1d 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -64,6 +64,11 @@ dockerdir: 'docker/barometer-ves' <<: *master <<: *other-receivers + - 'barometer-snmp': + project: 'barometer' + dockerdir: 'docker/barometer-snmp' + <<: *master + <<: *other-receivers - 'bottlenecks': project: 'bottlenecks' <<: *master @@ -86,6 +91,11 @@ dockerfile: 'docker/Dockerfile.local' <<: *master <<: *other-receivers + - 'qtip-nettest': + project: 'qtip' + dockerdir: 'contrib/nettest' + <<: *master + <<: *other-receivers - 'storperf-master': project: 'storperf' dockerdir: 'docker/storperf-master' diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index e9e929de6..0ac552082 100644 --- a/jjb/releng/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -117,7 +117,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' + project-pattern: 'functest|functest-kubernetes|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index b12f663aa..fb3bab42b 100644 --- a/jjb/releng/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml @@ -8,6 +8,7 @@ - 'archive-repositories' - 'check-status-of-slaves' - 'ansible-build-server' + - 'generate-artifacts-index-pages' ######################## # job templates @@ -88,7 +89,6 @@ name: 'check-status-of-slaves' disabled: false - concurrent: true parameters: @@ -96,17 +96,23 @@ name: SLAVE_NAME description: 'script lives on master node' default-slaves: - - master + - lf-build1 allowed-multiselect: false ignore-offline-nodes: true + - project-parameter: + project: releng + branch: master + + scm: + - git-scm triggers: - timed: '@midnight' builders: - shell: | - cd /opt/jenkins-ci/slavemonitor - bash slave-monitor-0.1.sh | sort + cd $WORKSPACE/utils/ + bash slave-monitor-0.1.sh - job-template: name: 'ansible-build-server' @@ -161,6 +167,43 @@ builders: - shell: | # run playbook - sudo ansible-playbook -C -D -i \ + sudo ansible-playbook -i \ $WORKSPACE/utils/build-server-ansible/inventory.ini \ $WORKSPACE/utils/build-server-ansible/main.yml + + +- job-template: + name: 'generate-artifacts-index-pages' + + project-type: freestyle + + disabled: false + + concurrent: false + + parameters: + - node: + name: SLAVE_NAME + description: Build Servers + default-slaves: + - lf-build2 + - project-parameter: + project: releng + branch: master + + scm: + - git-scm + + triggers: + - timed: '@hourly' + + builders: + - generate-artifact-html + + +- builder: + name: generate-artifact-html + builders: + - shell: | + cd $WORKSPACE/utils/ + ./artifacts.opnfv.org.sh diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index ef99f5abb..6fb6804bc 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -121,6 +121,4 @@ builders: - shell: | source /opt/virtualenv/jenkins-job-builder/bin/activate - cd /opt/jenkins-ci/releng - git pull jenkins-jobs update -r --delete-old jjb/ diff --git a/jjb/releng/automate.yml b/jjb/releng/testresults-automate.yml index 6bd952a1d..01fbe0c6d 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/testresults-automate.yml @@ -1,6 +1,6 @@ --- - project: - name: utils-automate + name: testresults-automate stream: - master: branch: '{stream}' @@ -17,7 +17,6 @@ jobs: - '{module}-automate-{stream}' - '{module}-automate-{phase}-{stream}' - - '{module}-verify-{stream}' project: 'releng-testresults' @@ -53,53 +52,6 @@ send-to-individuals: true - job-template: - name: '{module}-verify-{stream}' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: '{module}/**' - - builders: - - shell: | - cd {module}/ - tox - if [ -e *.xml ];then - cp *.xml $WORKSPACE - fi - - publishers: - - publish-coverage - - email-jenkins-admins-on-failure - -- job-template: name: '{module}-automate-{stream}' project-type: multijob @@ -135,21 +87,10 @@ fail: true triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: '{module}/**' + - gerrit-trigger-change-merged: + project: '**' + branch: '{branch}' + files: '{module}/**' builders: - description-setter: diff --git a/jjb/releng/testresults-verify.yml b/jjb/releng/testresults-verify.yml new file mode 100644 index 000000000..958833c0f --- /dev/null +++ b/jjb/releng/testresults-verify.yml @@ -0,0 +1,78 @@ +--- +- project: + name: testresults-verify + stream: + - master: + branch: '{stream}' + + module: + - 'testapi' + - 'reporting' + + jobs: + - '{module}-verify-{stream}' + - '{module}-client-verify-{stream}' + + project: 'releng-testresults' + +- job-template: + name: '{module}-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit-trigger-patchset-created: + server: 'gerrit.opnfv.org' + project: '**' + branch: '{branch}' + files: '{module}/**' + + builders: + - shell: | + cd {module}/ + tox + if [ -e *.xml ];then + cp *.xml $WORKSPACE + fi + + publishers: + - publish-coverage + - email-jenkins-admins-on-failure + +- job-template: + name: '{module}-client-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit-trigger-patchset-created: + server: 'gerrit.opnfv.org' + project: '**' + branch: '{branch}' + files: '{module}/{module}-client/**' + + builders: + - shell: | + cd {module}/{module}-client + tox + if [ -e *.xml ];then + cp *.xml $WORKSPACE + fi + + publishers: + - publish-coverage + - email-jenkins-admins-on-failure diff --git a/jjb/xci/bifrost-periodic-jobs.yml b/jjb/xci/bifrost-periodic-jobs.yml index 0bca26aa5..fbe2e205a 100644 --- a/jjb/xci/bifrost-periodic-jobs.yml +++ b/jjb/xci/bifrost-periodic-jobs.yml @@ -123,7 +123,7 @@ name: SLAVE_LABEL default: '{slave-label}' - string: - name: XCI_LOOP + name: CI_LOOP default: 'periodic' wrappers: diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml index e4c2d0e5b..2fb7c2167 100644 --- a/jjb/xci/bifrost-verify-jobs.yml +++ b/jjb/xci/bifrost-verify-jobs.yml @@ -71,7 +71,14 @@ use-build-blocker: true blocking-jobs: - 'xci-verify-{distro}-.*' + - 'xci-.*-{distro}-merge-.*' - '.*-bifrost-verify.*-{type}' + - throttle: + max-per-node: 2 + max-total: 10 + categories: + - xci-verify-virtual + option: category block-level: 'NODE' @@ -122,7 +129,7 @@ name: SLAVE_LABEL default: 'xci-virtual' - string: - name: XCI_LOOP + name: CI_LOOP default: 'verify' scm: @@ -147,7 +154,7 @@ wrappers: - fix-workspace-permissions - build-timeout: - timeout: 90 + timeout: 180 publishers: # yamllint disable rule:line-length @@ -155,6 +162,13 @@ recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com # yamllint enable rule:line-length - email-jenkins-admins-on-failure + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh + # ------------------------------- # trigger macros # ------------------------------- diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh index 451f33b4b..9219d0020 100755 --- a/jjb/xci/bifrost-verify.sh +++ b/jjb/xci/bifrost-verify.sh @@ -28,7 +28,7 @@ set -o pipefail cd ~/bifrost # provision 3 VMs; xcimaster, controller, and compute -./scripts/bifrost-provision.sh +./scripts/bifrost-provision.sh | ts sudo -H -E virsh list EOF @@ -46,6 +46,6 @@ export XCI_UPDATE_CLEAN_VM_OS=true ./xci/scripts/vm/start-new-vm.sh $VM_DISTRO -rsync -a -e "ssh -F $HOME/.ssh/xci-vm-config" $WORKSPACE/ ${VM_DISTRO}_xci_vm:~/bifrost +rsync -a -e "ssh -F $HOME/.ssh/${VM_DISTRO}-xci-vm-config" $WORKSPACE/ ${VM_DISTRO}_xci_vm:~/bifrost -ssh -F $HOME/.ssh/xci-vm-config ${VM_DISTRO}_xci_vm "cd ~/bifrost/releng-xci && ./bifrost_test.sh" +ssh -F $HOME/.ssh/${VM_DISTRO}-xci-vm-config ${VM_DISTRO}_xci_vm "cd ~/bifrost/releng-xci && ./bifrost_test.sh" diff --git a/jjb/xci/osa-periodic-jobs.yml b/jjb/xci/osa-periodic-jobs.yml index 64daa696b..6c2d165a4 100644 --- a/jjb/xci/osa-periodic-jobs.yml +++ b/jjb/xci/osa-periodic-jobs.yml @@ -216,6 +216,7 @@ export FUNCTEST_MODE=$FUNCTEST_MODE export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME export XCI_FLAVOR=$XCI_FLAVOR + export CORE_OPENSTACK_INSTALL=true export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ export INSTALLER_TYPE=$INSTALLER_TYPE @@ -234,7 +235,7 @@ - shell: | #!/bin/bash - ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" + ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" - builder: @@ -247,8 +248,8 @@ - shell: | #!/bin/bash - sudo virsh destroy ${DISTRO}_xci_vm - sudo virsh undefine ${DISTRO}_xci_vm + sudo virsh destroy ${DISTRO}_xci_vm || true + sudo virsh undefine ${DISTRO}_xci_vm || true # this will be enabled once the xci is prepared # - builder: diff --git a/jjb/xci/xci-cleanup.sh b/jjb/xci/xci-cleanup.sh index 45b4ec273..51a863da2 100755 --- a/jjb/xci/xci-cleanup.sh +++ b/jjb/xci/xci-cleanup.sh @@ -14,11 +14,15 @@ # what you are doing. #---------------------------------------------------------------------- +# Need to cover macros with and without parameters +VM_NAME=$DISTRO +VM_NAME+=_xci_vm + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then echo "Skipping the deployment!" exit 0 fi -sudo virsh destroy ${DISTRO}_xci_vm -sudo virsh undefine ${DISTRO}_xci_vm +sudo virsh destroy $VM_NAME || true +sudo virsh undefine $VM_NAME || true diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml index a95374933..11db4e168 100644 --- a/jjb/xci/xci-daily-jobs.yml +++ b/jjb/xci/xci-daily-jobs.yml @@ -113,7 +113,7 @@ name: SLAVE_LABEL default: '{slave-label}' - string: - name: XCI_LOOP + name: CI_LOOP default: 'daily' triggers: @@ -131,7 +131,7 @@ predefined-parameters: | DEPLOY_SCENARIO=$DEPLOY_SCENARIO XCI_FLAVOR=$XCI_FLAVOR - XCI_LOOP=$XCI_LOOP + CI_LOOP=$CI_LOOP same-node: true block: true - trigger-builds: @@ -140,7 +140,7 @@ predefined-parameters: | DEPLOY_SCENARIO=$DEPLOY_SCENARIO XCI_FLAVOR=$XCI_FLAVOR - XCI_LOOP=$XCI_LOOP + CI_LOOP=$CI_LOOP same-node: true block: true block-thresholds: @@ -154,6 +154,12 @@ recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com # yamllint enable rule:line-length - email-jenkins-admins-on-failure + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh - job-template: name: 'xci-{phase}-{pod}-{distro}-daily-{stream}' @@ -214,7 +220,7 @@ name: SLAVE_LABEL default: '{slave-label}' - string: - name: XCI_LOOP + name: CI_LOOP default: 'daily' - string: name: INSTALLER_TYPE diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh index 211d282d3..71cf96d8d 100755 --- a/jjb/xci/xci-deploy.sh +++ b/jjb/xci/xci-deploy.sh @@ -15,14 +15,14 @@ cd $WORKSPACE/xci # for daily jobs, we want to use working versions # for periodic jobs, we will use whatever is set in the job, probably master -if [[ "$XCI_LOOP" == "daily" ]]; then +if [[ "$CI_LOOP" == "daily" ]]; then # source pinned-vars to get releng version source ./config/pinned-versions # checkout the version git checkout -q $OPNFV_RELENG_VERSION echo "Info: Using $OPNFV_RELENG_VERSION" -elif [[ "$XCI_LOOP" == "periodic" ]]; then +elif [[ "$CI_LOOP" == "periodic" ]]; then echo "Info: Using $OPNFV_RELENG_VERSION" fi @@ -31,7 +31,7 @@ fi # to take this into account while deploying anyways # clone openstack-ansible # stable/ocata already use pinned versions so this is only valid for master -if [[ "$XCI_LOOP" == "periodic" && "$OPENSTACK_OSA_VERSION" == "master" ]]; then +if [[ "$CI_LOOP" == "periodic" && "$OPENSTACK_OSA_VERSION" == "master" ]]; then cd $WORKSPACE # get the url to openstack-ansible git source ./config/env-vars diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml index b8da6128a..92eea190f 100644 --- a/jjb/xci/xci-merge-jobs.yml +++ b/jjb/xci/xci-merge-jobs.yml @@ -61,8 +61,7 @@ blocking-jobs: - 'xci-verify-{distro}-.*' - 'xci-.*-{distro}-merge-.*' - - 'bifrost-verify-{distro}-.*' - - 'bifrost-periodic-{distro}-.*' + - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' - 'xci-(os|k8s).*?-virtual-{distro}-.*' @@ -71,7 +70,7 @@ max-per-node: 2 max-total: 10 categories: - - xci-merge-virtual + - xci-verify-virtual option: category wrappers: @@ -168,6 +167,14 @@ kill-phase-on: NEVER abort-all-job: true + publishers: + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh + - job-template: name: 'xci-{phase}-{type}-{distro}-merge-{stream}' @@ -254,8 +261,6 @@ builders: - shell: !include-raw: ./xci-run-functest.sh - - shell: - !include-raw: ./xci-cleanup.sh - builder: name: 'xci-merge-promote-macro' diff --git a/jjb/xci/xci-promote.sh b/jjb/xci/xci-promote.sh index 98ad7ff4a..62818abee 100755 --- a/jjb/xci/xci-promote.sh +++ b/jjb/xci/xci-promote.sh @@ -14,6 +14,9 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the healthcheck if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the healthcheck!" diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh index b2421de32..60b48cf7f 100755 --- a/jjb/xci/xci-run-functest.sh +++ b/jjb/xci/xci-run-functest.sh @@ -7,6 +7,9 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +set -o errexit +set -o nounset +set -o pipefail #---------------------------------------------------------------------- # This script is used by CI and executed by Jenkins jobs. @@ -14,12 +17,33 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the healthcheck if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the healthcheck!" exit 0 fi +# skip the healthcheck if the scenario is Kubernetes scenario +if [[ "$DEPLOY_SCENARIO" =~ k8 ]]; then + echo "Skipping the healthcheck!" + exit 0 +fi + +# if the scenario is external, we need to wipe WORKSPACE to place releng-xci there since +# the project where the scenario is coming from is cloned and the patch checked out to the +# xci/scenarios/$DEPLOY_SCENARIO to be synched on clean VM +# apart from that, we need releng-xci stuff in WORKSPACE for things to function correctly on Jenkins. +# if the change is coming to releng-xci, we don't need to do anything since the patch is checked +# out to the WORKSPACE anyways +if [[ $GERRIT_PROJECT != "releng-xci" ]]; then + cd $HOME && /bin/rm -rf $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + chmod -R go-rwx $WORKSPACE/xci/scripts/vm +fi + # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then @@ -27,4 +51,10 @@ if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENT exit 0 fi -ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci/xci && PATH=/home/devuser/.local/bin:$PATH ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/run-functest.yml" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci/xci && PATH=/home/devuser/.local/bin:$PATH ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/prepare-functest.yml" +echo "Running functest" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "/root/run-functest.sh" +echo "Functest log" +echo "---------------------------------------------------------------------------------" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "cat /root/results/functest.log" +echo "---------------------------------------------------------------------------------" diff --git a/jjb/xci/xci-set-scenario.sh b/jjb/xci/xci-set-scenario.sh new file mode 100755 index 000000000..62b26c019 --- /dev/null +++ b/jjb/xci/xci-set-scenario.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o pipefail + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# This function allows developers to specify the impacted scenario by adding +# the info about installer and scenario into the commit message or using +# the topic branch names. This results in either skipping the real verification +# totally or skipping the determining the installer and scenario programmatically. +# It is important to note that this feature is only available to generic scenarios +# and only single installer/scenario pair is allowed. +# The input in commit message should be placed at the end of the commit message body, +# before the signed-off and change-id lines. +# +# Pattern to be searched in Commit Message +# deploy-scenario:<scenario-name> +# installer-type:<installer-type> +# Examples: +# deploy-scenario:os-odl-nofeature +# installer-type:osa +# +# deploy-scenario:k8-nosdn-nofeature +# installer-type:kubespray +# +# Patterns to be searched in topic branch name +# skip-verify +# skip-deployment +# force-verify +function override_generic_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # process topic branch names + if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then + # skip the real verification + echo "Skipping verify!" + echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + exit 0 + elif [[ "$GERRIT_TOPIC" =~ 'force-verify' ]]; then + # Run the deployment with default installer and scenario when multiple things change + # and we want to force that. + echo "Recording the installer 'osa' and scenario 'os-nosdn-nofeature' for downstream jobs" + echo "Forcing CI verification of default scenario and installer!" + echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + exit 0 + fi + + # process commit message + if [[ "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "installer-type:" && "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "deploy-scenario:" ]]; then + INSTALLER_TYPE=$(echo $GERRIT_CHANGE_COMMIT_MESSAGE | awk '/installer-type:/' RS=" " | cut -d":" -f2) + DEPLOY_SCENARIO=$(echo $GERRIT_CHANGE_COMMIT_MESSAGE | awk '/deploy-scenario:/' RS=" " | cut -d":" -f2) + + if [[ -z "$INSTALLER_TYPE" || -z "$DEPLOY_SCENARIO" ]]; then + echo "Installer type or deploy scenario is not specified. Falling back to programmatically determining them." + else + echo "Recording the installer '$INSTALLER_TYPE' and scenario '$DEPLOY_SCENARIO' for downstream jobs" + echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties + exit 0 + fi + else + echo "Installer type or deploy scenario is not specified. Falling back to programmatically determining them." + fi +} + +# This function determines the impacted generic scenario by processing the +# change and using diff to see what changed. If changed files belong to a scenario +# its name gets recorded for deploying and testing the right scenario. +# +# Pattern to be searched in Changeset +# releng-xci/scenarios/<scenario>/<impacted files>: <scenario> +# releng-xci/xci/installer/osa/<impacted files>: os-nosdn-nofeature +# releng-xci/xci/installer/kubespray/<impacted files>: k8-nosdn-nofeature +# the rest: os-nosdn-nofeature +function determine_generic_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # get the changeset + cd $WORKSPACE + CHANGESET=$(git diff HEAD^..HEAD --name-only) + for CHANGED_FILE in $CHANGESET; do + case $CHANGED_FILE in + *k8-nosdn*|*kubespray*) + [[ ${DEPLOY_SCENARIO[@]} =~ "k8-nosdn-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='k8-nosdn-nofeature' + ;; + *os-odl*) + [[ ${DEPLOY_SCENARIO[@]} =~ "os-odl-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='os-odl-nofeature' + ;; + *os-nosdn*|*osa*) + [[ ${DEPLOY_SCENARIO[@]} =~ "os-nosdn-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='os-nosdn-nofeature' + ;; + *) + [[ ${DEPLOY_SCENARIO[@]} =~ "os-nosdn-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='os-nosdn-nofeature' + ;; + esac + done +} + +# This function determines the impacted external scenario by processing the Gerrit +# change and using diff to see what changed. If changed files belong to a scenario +# its name gets recorded for deploying and testing the right scenario. +# +# Pattern +# <project-repo>/scenarios/<scenario>/<impacted files>: <scenario> +function determine_external_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # remove the clone that is done via jenkins and place releng-xci there so the + # things continue functioning properly + cd $HOME && /bin/rm -rf $WORKSPACE + git clone -q https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + + # fix the permissions so ssh doesn't complain due to having world-readable keyfiles + chmod -R go-rwx $WORKSPACE/xci/scripts/vm + + # clone the project repo and fetch the patchset to process for further processing + git clone -q https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT $WORK_DIRECTORY/$GERRIT_PROJECT + cd $WORK_DIRECTORY/$GERRIT_PROJECT + git fetch -q https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT $GERRIT_REFSPEC && git checkout -q FETCH_HEAD + + # process the diff to find out what scenario(s) are impacted - there should only be 1 + DEPLOY_SCENARIO+=$(git diff HEAD^..HEAD --name-only | grep scenarios | awk -F '[/|/]' '{print $2}' | uniq) +} + +echo "Determining the impacted scenario" + +declare -a DEPLOY_SCENARIO + +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + +# this directory is where the temporary clones and files are created +# while extracting the impacted scenario +WORK_DIRECTORY=/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO +/bin/rm -rf $WORK_DIRECTORY && mkdir -p $WORK_DIRECTORY + +if [[ $GERRIT_PROJECT == "releng-xci" ]]; then + override_generic_scenario + determine_generic_scenario +else + determine_external_scenario +fi + +# ensure single scenario is impacted + if [[ $(IFS=$'\n' echo ${DEPLOY_SCENARIO[@]} | wc -w) != 1 ]]; then + echo "Change impacts multiple scenarios!" + echo "XCI doesn't support testing of changes that impact multiple scenarios currently." + echo "Please split your change into multiple different/dependent changes, each modifying single scenario." + exit 1 +fi + +# set the installer +case ${DEPLOY_SCENARIO[0]} in + os-*) + INSTALLER_TYPE=osa + ;; + k8-*) + INSTALLER_TYPE=kubespray + ;; + *) + echo "Unable to determine the installer. Exiting!" + exit 1 + ;; +esac + +# save the installer and scenario names into java properties file +# so they can be injected to downstream jobs via envInject +echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' for downstream jobs" +echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties +echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties + +# skip the deployment if the scenario is not supported on this distro +OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml +if ! sed -n "/^- scenario: ${DEPLOY_SCENARIO[0]}$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then + echo "# SKIPPED: Scenario ${DEPLOY_SCENARIO[0]} is NOT supported on $DISTRO" + exit 0 +fi diff --git a/jjb/xci/xci-start-deployment.sh b/jjb/xci/xci-start-deployment.sh index e86f5b3a1..102ca41c3 100755 --- a/jjb/xci/xci-start-deployment.sh +++ b/jjb/xci/xci-start-deployment.sh @@ -14,12 +14,27 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the deployment!" exit 0 fi +# if the scenario is external, we need to wipe WORKSPACE to place releng-xci there since +# the project where the scenario is coming from is cloned and the patch checked out to the +# xci/scenarios/$DEPLOY_SCENARIO to be synched on clean VM +# apart from that, we need releng-xci stuff in WORKSPACE for things to function correctly on Jenkins. +# if the change is coming to releng-xci, we don't need to do anything since the patch is checked +# out to the WORKSPACE anyways +if [[ $GERRIT_PROJECT != "releng-xci" ]]; then + cd $HOME && /bin/rm -rf $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + chmod -R go-rwx $WORKSPACE/xci/scripts/vm +fi + # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then @@ -27,4 +42,4 @@ if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENT exit 0 fi -ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" diff --git a/jjb/xci/xci-start-new-vm.sh b/jjb/xci/xci-start-new-vm.sh index b72c33920..7874e6776 100755 --- a/jjb/xci/xci-start-new-vm.sh +++ b/jjb/xci/xci-start-new-vm.sh @@ -14,12 +14,27 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then echo "Skipping the deployment!" exit 0 fi +# if the scenario is external, we need to wipe WORKSPACE to place releng-xci there since +# the project where the scenario is coming from is cloned and the patch checked out to the +# xci/scenarios/$DEPLOY_SCENARIO to be synched on clean VM +# apart from that, we need releng-xci stuff in WORKSPACE for things to function correctly on Jenkins. +# if the change is coming to releng-xci, we don't need to do anything since the patch is checked +# out to the WORKSPACE anyways +if [[ $GERRIT_PROJECT != "releng-xci" ]]; then + cd $HOME && /bin/rm -rf $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + chmod -R go-rwx $WORKSPACE/xci/scripts/vm +fi + # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then @@ -35,11 +50,14 @@ cd $WORKSPACE # yourself. cat > xci_test.sh<<EOF #!/bin/bash +set -o pipefail export DISTRO=$DISTRO export DEPLOY_SCENARIO=$DEPLOY_SCENARIO export FUNCTEST_MODE=$FUNCTEST_MODE export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME export XCI_FLAVOR=$XCI_FLAVOR +export CORE_OPENSTACK_INSTALL=true +export BIFROST_USE_PREBUILT_IMAGES=true export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ export INSTALLER_TYPE=$INSTALLER_TYPE @@ -53,7 +71,7 @@ if [[ ! -z ${WORKSPACE+x} && $GERRIT_PROJECT != "releng-xci" ]]; then fi cd xci -./xci-deploy.sh +./xci-deploy.sh | ts EOF chmod a+x xci_test.sh diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 7c6871c22..4c9ff9e69 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -67,8 +67,7 @@ blocking-jobs: - 'xci-verify-{distro}-.*' - 'xci-.*-{distro}-merge-.*' - - 'bifrost-verify-{distro}-.*' - - 'bifrost-periodic-{distro}-.*' + - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' block-level: 'NODE' @@ -103,7 +102,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'ANT' - project-pattern: '{project}' + project-pattern: 'releng-xci' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -111,6 +110,15 @@ forbidden-file-paths: - compare-type: ANT pattern: 'xci/scripts/vm/**' + - project-compare-type: 'REG_EXP' + project-pattern: 'sfc|sdnvpn' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + file-paths: + - compare-type: ANT + pattern: 'scenarios/**' readable-message: true custom-url: '* $JOB_NAME $BUILD_URL' skip-vote: @@ -121,12 +129,15 @@ parameters: - project-parameter: - project: '{project}' + project: $GERRIT_PROJECT branch: '{branch}' - label: name: SLAVE_LABEL default: 'xci-virtual' - string: + name: DISTRO + default: '{distro}' + - string: name: CLEAN_DIB_IMAGES default: 'true' - string: @@ -135,8 +146,11 @@ description: 'Git URL to use on this Jenkins Slave' builders: + - 'xci-verify-set-scenario-macro' + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - multijob: name: deploy condition: SUCCESSFUL @@ -145,8 +159,8 @@ current-parameters: true predefined-parameters: | DISTRO={distro} - DEPLOY_SCENARIO=os-nosdn-nofeature CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -162,18 +176,27 @@ current-parameters: true predefined-parameters: | DISTRO={distro} - DEPLOY_SCENARIO=os-nosdn-nofeature CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + FUNCTEST_MODE=tier + FUNCTEST_TIER=healthcheck node-parameters: true kill-phase-on: NEVER abort-all-job: true + publishers: + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh + + - job-template: name: 'xci-verify-{distro}-{phase}-{type}-{stream}' @@ -195,14 +218,11 @@ parameters: - string: name: PROJECT - default: 'releng-xci' + default: $GERRIT_PROJECT - string: name: DISTRO default: 'ubuntu' - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature' - - string: name: FUNCTEST_MODE default: 'tier' - string: @@ -218,14 +238,13 @@ name: OPNFV_RELENG_DEV_PATH default: $WORKSPACE/ - string: - name: INSTALLER_TYPE - default: 'osa' - - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - ssh-agent-wrapper - build-timeout: timeout: 240 @@ -236,13 +255,19 @@ builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - 'xci-verify-{phase}-macro' # ------------------------------- # builder macros # ------------------------------- - builder: + name: 'xci-verify-set-scenario-macro' + builders: + - shell: + !include-raw: ./xci-set-scenario.sh + +- builder: name: 'xci-verify-deploy-macro' builders: - shell: @@ -255,4 +280,4 @@ name: 'xci-verify-healthcheck-macro' builders: - shell: - !include-raw: ./xci-cleanup.sh + !include-raw: ./xci-run-functest.sh diff --git a/jjb/yardstick/yardstick-daily-jobs.yml b/jjb/yardstick/yardstick-daily-jobs.yml index 481af2a42..20890fa13 100644 --- a/jjb/yardstick/yardstick-daily-jobs.yml +++ b/jjb/yardstick/yardstick-daily-jobs.yml @@ -136,7 +136,7 @@ <<: *euphrates # daisy CI PODs - baremetal: - slave-label: zte-pod3 + slave-label: daisy-baremetal installer: daisy auto-trigger-name: 'daily-trigger-disabled' <<: *master diff --git a/jjb/yardstick/yardstick-daily.sh b/jjb/yardstick/yardstick-daily.sh index f0e4f8e22..2fd68484e 100755 --- a/jjb/yardstick/yardstick-daily.sh +++ b/jjb/yardstick/yardstick-daily.sh @@ -50,7 +50,7 @@ opts="--privileged=true --rm" envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ -e NODE_NAME=${NODE_NAME} -e EXTERNAL_NETWORK=${EXTERNAL_NETWORK} \ -e YARDSTICK_BRANCH=${BRANCH} -e BRANCH=${BRANCH} \ - -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" + -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} -e CI_DEBUG=true" if [[ "${INSTALLER_TYPE}" == 'fuel' ]]; then envs+=" -e SSH_KEY=/root/.ssh/mcp.rsa" diff --git a/jjb/yardstick/yardstick-project-jobs.yml b/jjb/yardstick/yardstick-project-jobs.yml index e2fee2988..c03666388 100644 --- a/jjb/yardstick/yardstick-project-jobs.yml +++ b/jjb/yardstick/yardstick-project-jobs.yml @@ -132,7 +132,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running unit tests in Python 2.7 ..." cd $WORKSPACE @@ -146,7 +148,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running unit tests in Python 3 ..." cd $WORKSPACE @@ -160,7 +164,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running functional tests in Python 2.7 ..." cd $WORKSPACE @@ -174,7 +180,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running functional tests in Python 3 ..." cd $WORKSPACE @@ -188,7 +196,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running coverage tests ..." cd $WORKSPACE @@ -202,7 +212,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running style guidelines (PEP8) tests ..." cd $WORKSPACE |