diff options
72 files changed, 2847 insertions, 665 deletions
diff --git a/jjb/apex/apex-verify-jobs.yml b/jjb/apex/apex-verify-jobs.yml index 88c1b1765..302932246 100644 --- a/jjb/apex/apex-verify-jobs.yml +++ b/jjb/apex/apex-verify-jobs.yml @@ -108,8 +108,6 @@ - job-template: name: 'apex-verify-{stream}' - node: 'apex-virtual-master' - concurrent: true disabled: '{obj:disabled}' @@ -117,6 +115,7 @@ project-type: 'multijob' parameters: + - '{project}-virtual-{stream}-defaults' - apex-parameter: gs-pathname: '{gs-pathname}/dev' - project-parameter: @@ -248,8 +247,6 @@ - job-template: name: 'apex-verify-gate-{stream}' - node: 'apex-build-{stream}' - concurrent: true disabled: '{obj:disabled}' @@ -257,6 +254,7 @@ project-type: 'multijob' parameters: + - '{project}-virtual-{stream}-defaults' - apex-parameter: gs-pathname: '{gs-pathname}/dev' - project-parameter: diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 6714d6a66..cf29b928c 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -112,37 +112,33 @@ <<: *master - 'os-odl-bgpvpn-noha': <<: *master + - 'os-odl-sriov-noha': + <<: *master + - 'os-odl-ovs_offload-noha': + <<: *master - 'os-ovn-nofeature-noha': <<: *master - 'os-nosdn-fdio-noha': <<: *master - 'os-nosdn-fdio-ha': <<: *master - - 'os-odl-fdio-noha': - <<: *master - - 'os-odl-fdio-ha': - <<: *master - 'os-nosdn-bar-ha': <<: *master - 'os-nosdn-bar-noha': <<: *master - - 'os-nosdn-nofeature-ha-ipv6': - <<: *master - 'os-nosdn-ovs_dpdk-noha': <<: *master - 'os-nosdn-ovs_dpdk-ha': <<: *master - - 'os-nosdn-kvm_ovs_dpdk-noha': + - 'os-odl-ovs_dpdk-noha': <<: *master - - 'os-nosdn-kvm_ovs_dpdk-ha': + - 'os-odl-ovs_dpdk-ha': <<: *master - 'os-odl-sfc-noha': <<: *master - 'os-odl-sfc-ha': <<: *master - - 'os-odl-fdio_dvr-noha': - <<: *master - - 'os-odl-fdio_dvr-ha': + - 'os-nosdn-calipso-noha': <<: *master - 'os-nosdn-nofeature-noha': <<: *euphrates @@ -255,8 +251,6 @@ - job-template: name: 'apex-deploy-{platform}-{stream}' - node: 'apex-{platform}-{stream}' - concurrent: true disabled: false @@ -315,10 +309,10 @@ - job-template: name: 'apex-virtual-{stream}' - node: 'apex-virtual-master' - project-type: 'multijob' + concurrent: true + disabled: false scm: @@ -581,8 +575,6 @@ # branch: branch (eg. stable) project-type: 'multijob' - node: '{baremetal-slave}' - disabled: '{obj:disable_daily}' scm: @@ -1345,7 +1337,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-ovn-nofeature-noha-baremetal-master' + - name: 'apex-os-odl-sriov-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1353,7 +1345,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-fdio-noha-baremetal-master' + - name: 'apex-os-odl-ovs_offload-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1361,7 +1353,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-fdio-ha-baremetal-master' + - name: 'apex-os-ovn-nofeature-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1369,7 +1361,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-fdio-noha-baremetal-master' + - name: 'apex-os-nosdn-fdio-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1377,7 +1369,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-fdio-ha-baremetal-master' + - name: 'apex-os-nosdn-fdio-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1401,14 +1393,6 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-nofeature-ha-ipv6-baremetal-master' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - name: 'apex-os-nosdn-ovs_dpdk-noha-baremetal-master' node-parameters: false current-parameters: false @@ -1425,7 +1409,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-kvm_ovs_dpdk-noha-baremetal-master' + - name: 'apex-os-odl-ovs_dpdk-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1433,7 +1417,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-kvm_ovs_dpdk-ha-baremetal-master' + - name: 'apex-os-odl-ovs_dpdk-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1457,15 +1441,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-fdio_dvr-noha-baremetal-master' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - - name: 'apex-os-odl-fdio_dvr-ha-baremetal-master' + - name: 'apex-os-nosdn-calipso-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index 27a854dd7..ab65c4e2b 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -129,8 +129,6 @@ - job-template: name: 'apex-deploy-{platform}-{stream}' - node: 'apex-{platform}-{stream}' - concurrent: true disabled: false @@ -189,10 +187,10 @@ - job-template: name: 'apex-virtual-{stream}' - node: 'apex-virtual-master' - project-type: 'multijob' + concurrent: true + disabled: false scm: @@ -455,8 +453,6 @@ # branch: branch (eg. stable) project-type: 'multijob' - node: '{baremetal-slave}' - disabled: '{obj:disable_daily}' scm: diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden index 789ca7f7e..93ce931eb 100644 --- a/jjb/apex/scenarios.yaml.hidden +++ b/jjb/apex/scenarios.yaml.hidden @@ -5,22 +5,20 @@ master: - 'os-odl-nofeature-noha' - 'os-odl-bgpvpn-ha' - 'os-odl-bgpvpn-noha' + - 'os-odl-sriov-noha' + - 'os-odl-ovs_offload-noha' - 'os-ovn-nofeature-noha' - 'os-nosdn-fdio-noha' - 'os-nosdn-fdio-ha' - - 'os-odl-fdio-noha' - - 'os-odl-fdio-ha' - 'os-nosdn-bar-ha' - 'os-nosdn-bar-noha' - - 'os-nosdn-nofeature-ha-ipv6' - 'os-nosdn-ovs_dpdk-noha' - 'os-nosdn-ovs_dpdk-ha' - - 'os-nosdn-kvm_ovs_dpdk-noha' - - 'os-nosdn-kvm_ovs_dpdk-ha' + - 'os-odl-ovs_dpdk-noha' + - 'os-odl-ovs_dpdk-ha' - 'os-odl-sfc-noha' - 'os-odl-sfc-ha' - - 'os-odl-fdio_dvr-noha' - - 'os-odl-fdio_dvr-ha' + - 'os-nosdn-calipso-noha' euphrates: - 'os-nosdn-nofeature-noha' - 'os-nosdn-nofeature-ha' diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index da3992ea8..1c5f14880 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -62,11 +62,19 @@ auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' - 'os-odl-nofeature-ha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + - 'os-ovn-nofeature-ha': + auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-ovs-ha': + auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' # NOHA scenarios - 'os-nosdn-nofeature-noha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + exclude: + - scenario: os-ovn-nofeature-ha + stream: euphrates + jobs: - '{installer}-{scenario}-{pod}-daily-{stream}' - '{installer}-deploy-{pod}-daily-{stream}' @@ -122,15 +130,51 @@ DEPLOY_SCENARIO={scenario} same-node: true block: true + - trigger-builds: + - project: 'functest-{installer}-{pod}-arm-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - trigger-builds: + - project: 'yardstick-{installer}-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + # 1.here the stream means the SUT stream, dovetail stream is defined in its own job + # 2.testsuite proposed_tests here is for new test cases planning to add into OVP + # 3.run proposed_tests on Monday, Wednesday and Friday against ha scenario + # 4.testsuite default here is for the test cases already added into OVP + # 5.run default testsuite on Tuesday against ha scenario + # 6.not used for release criteria or compliance, + # only to debug the dovetail tool bugs with arm pods - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + MON: true + WED: true + FRI: true + use-build-time: true steps: - trigger-builds: - - project: 'functest-{installer}-{pod}-arm-daily-{stream}' + - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -141,14 +185,19 @@ failure-threshold: 'never' unstable-threshold: 'FAILURE' - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + TUES: true + use-build-time: true steps: - trigger-builds: - - project: 'yardstick-{installer}-{pod}-daily-{stream}' + - project: 'dovetail-{installer}-{pod}-default-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -158,21 +207,6 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.here the stream means the SUT stream, dovetail stream is defined in its own job - # 2.only debug testsuite here(refstack, ha, vping, ipv6, tempest, bgpvpn) - # 3.not used for release criteria or compliance, - # only to debug the dovetail tool bugs with arm pods - - trigger-builds: - - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' # Armband uses Fuel's log collection project job, no need to duplicate - conditional-step: condition-kind: not @@ -250,7 +284,7 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 2,4,6' + - timed: '0 1 * * 4,6' - trigger: name: 'fuel-os-nosdn-nofeature-noha-armband-baremetal-master-trigger' triggers: @@ -258,7 +292,15 @@ - trigger: name: 'fuel-os-odl-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 1,3,5,7' + - timed: '0 1 * * 2,5,7' +- trigger: + name: 'fuel-os-ovn-nofeature-ha-armband-baremetal-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-baremetal-master-trigger' + triggers: + - timed: '0 1 * * 1,3' # --------------------------------------------------------------------- # Enea Armband CI Baremetal Triggers running against euphrates branch # --------------------------------------------------------------------- @@ -274,6 +316,11 @@ name: 'fuel-os-odl-nofeature-ha-armband-baremetal-euphrates-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-baremetal-euphrates-trigger' + triggers: + - timed: '' + # -------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against master branch # -------------------------------------------------------------- @@ -289,6 +336,14 @@ name: 'fuel-os-odl-nofeature-ha-armband-virtual-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-ovn-nofeature-ha-armband-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-virtual-master-trigger' + triggers: + - timed: '' # ------------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against euphrates branch # ------------------------------------------------------------------- @@ -304,3 +359,7 @@ name: 'fuel-os-odl-nofeature-ha-armband-virtual-euphrates-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-armband-virtual-euphrates-trigger' + triggers: + - timed: '' diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 70296bda1..357f24e37 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -123,6 +123,15 @@ - 'os-nosdn-bar-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-bar-noha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'k8-nosdn-stor4nfv-ha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'k8-nosdn-stor4nfv-noha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' jobs: - 'compass-{scenario}-{pod}-daily-{stream}' @@ -149,7 +158,9 @@ use-build-blocker: true blocking-jobs: - 'compass-os-.*?-{pod}-daily-.*?' + - 'compass-k8-.*?-{pod}-daily-.*?' - 'compass-os-.*?-baremetal-daily-.*?' + - 'compass-k8-.*?-baremetal-daily-.*?' - 'compass-verify-[^-]*-[^-]*' block-level: 'NODE' @@ -166,9 +177,7 @@ installer: '{installer}' gs-pathname: '{gs-pathname}' ppa-pathname: '{ppa-pathname}' - - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + scenario: '{scenario}' - '{slave-label}-defaults' - '{installer}-defaults' @@ -209,12 +218,10 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # dovetail only master by now, not sync with A/B/C branches # here the stream means the SUT stream, dovetail stream is defined in its own job - # only run on os-(nosdn|odl_l2|odl_l3)-nofeature-ha scenario - # run against SUT master/euphrates branch, dovetail docker image with latest tag(Monday, Tuesday) - # run against SUT master/euphrates branch, dovetail docker image with cvp.X.X.X tag(Thursday, Friday) - # run against SUT danube branch, dovetail docker image with cvp.X.X.X tag on huawei-pod7 + # only run on os-(nosdn|odl_l3)-nofeature-ha scenario + # run with testsuite default, dovetail docker image with latest tag(Monday, Tuesday) + # run with testsuite proposed_tests, dovetail docker image with latest tag(Thursday, Friday) - conditional-step: condition-kind: and condition-operands: @@ -229,7 +236,7 @@ use-build-time: true steps: - trigger-builds: - - project: 'dovetail-compass-{pod}-proposed_tests-{stream}' + - project: 'dovetail-compass-{pod}-default-{stream}' current-parameters: false predefined-parameters: | DOCKER_TAG=latest @@ -338,6 +345,7 @@ installer: '{installer}' gs-pathname: '{gs-pathname}' ppa-pathname: '{ppa-pathname}' + scenario: '{scenario}' - '{slave-label}-defaults' - '{installer}-defaults' @@ -384,6 +392,7 @@ installer: '{installer}' gs-pathname: '{gs-pathname}' ppa-pathname: '{ppa-pathname}' + scenario: '{scenario}' - '{slave-label}-defaults' - '{installer}-defaults' @@ -429,6 +438,17 @@ name: LOG_DIRECTORY default: $WORKSPACE/log_output description: "Directory where the logs will be located upon the completion of the collection." + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + description: "Scenario to deploy with." + - string: + name: COMPASS_OS_VERSION + default: '' + - string: + name: COMPASS_OPENSTACK_VERSION + default: '' + ######################## # trigger macros @@ -484,11 +504,15 @@ - trigger: name: 'compass-k8-nosdn-nofeature-ha-baremetal-centos-master-trigger' triggers: - - timed: '' + - timed: '0 6 * * *' - trigger: name: 'compass-os-nosdn-bar-ha-baremetal-centos-master-trigger' triggers: - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-centos-master-trigger' + triggers: + - timed: '' # ---------------------------- # noha-baremetal-centos-master @@ -517,6 +541,15 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-baremetal-centos-master-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-baremetal-centos-master-trigger' + triggers: + - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-baremetal-centos-master-trigger' + triggers: + - timed: '' + # -------------------------- # ha-huawei-pod7-danube @@ -573,6 +606,10 @@ name: 'compass-os-nosdn-bar-ha-huawei-pod7-danube-trigger' triggers: - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # ---------------------------- # noha-huawei-pod7-danube @@ -601,6 +638,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-huawei-pod7-danube-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # '0 19 * * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # ------------------- # ha-baremetal-master @@ -648,7 +693,7 @@ - trigger: name: 'compass-k8-nosdn-nofeature-ha-baremetal-master-trigger' triggers: - - timed: '' + - timed: '0 10 2-30/2 * *' - trigger: name: 'compass-os-odl-sfc-ha-baremetal-master-trigger' triggers: @@ -657,6 +702,10 @@ name: 'compass-os-nosdn-bar-ha-baremetal-master-trigger' triggers: - timed: '0 2 2-30/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-master-trigger' + triggers: + - timed: '0 16 1-29/2 * *' # --------------------- # noha-baremetal-master @@ -685,6 +734,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-baremetal-master-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-baremetal-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-baremetal-master-trigger' + triggers: + - timed: '' # ------------------- # ha-baremetal-euphrates @@ -741,6 +798,10 @@ name: 'compass-os-nosdn-bar-ha-baremetal-euphrates-trigger' triggers: - timed: '0 21 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-euphrates-trigger' + triggers: + - timed: '' # --------------------- # noha-baremetal-euphrates @@ -769,6 +830,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-baremetal-euphrates-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-bar-noha-baremetal-euphrates-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-baremetal-euphrates-trigger' + triggers: + - timed: '' # ----------------- # ha-virtual-master @@ -825,6 +894,10 @@ name: 'compass-os-nosdn-bar-ha-virtual-master-trigger' triggers: - timed: '0 17 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-virtual-master-trigger' + triggers: + - timed: '0 15 2-30/2 * *' # ------------------- # noha-virtual-master @@ -853,6 +926,14 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-virtual-master-trigger' triggers: - timed: '0 11 2-30/2 * *' +- trigger: + name: 'compass-os-nosdn-bar-noha-virtual-master-trigger' + triggers: + - timed: '0 22 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-virtual-master-trigger' + triggers: + - timed: '0 10 2-30/2 * *' # ----------------- # ha-virtual-euphrates @@ -909,6 +990,10 @@ name: 'compass-os-nosdn-bar-ha-virtual-euphrates-trigger' triggers: - timed: '0 19 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-ha-virtual-euphrates-trigger' + triggers: + - timed: '' # '0 14 1-29/2 * *' # ------------------- # noha-virtual-euphrates @@ -937,3 +1022,11 @@ name: 'compass-os-nosdn-ovs_dpdk-noha-virtual-euphrates-trigger' triggers: - timed: '0 12 2-30/2 * *' +- trigger: + name: 'compass-os-nosdn-bar-noha-virtual-euphrates-trigger' + triggers: + - timed: '0 12 1-29/2 * *' +- trigger: + name: 'compass-k8-nosdn-stor4nfv-noha-virtual-euphrates-trigger' + triggers: + - timed: '' # '0 12 2-30/2 * *' diff --git a/jjb/compass4nfv/compass-logs.sh b/jjb/compass4nfv/compass-logs.sh index 03c8b20ac..c028194e7 100644 --- a/jjb/compass4nfv/compass-logs.sh +++ b/jjb/compass4nfv/compass-logs.sh @@ -14,7 +14,7 @@ OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") COMPASS_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_${OPNFV_ARTIFACT_VERSION}.log.tar.gz" -sudo docker exec -it compass-tasks /bin/bash /opt/collect-log.sh +sudo docker exec compass-tasks /bin/bash /opt/collect-log.sh sudo docker cp compass-tasks:/opt/log.tar.gz ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME} sudo chown $(whoami):$(whoami) ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME} diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index f215b78ac..444b17361 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -70,6 +70,7 @@ blocking-jobs: - 'compass-verify-[^-]*-[^-]*' - 'compass-os-.*?-virtual-daily-.*?' + - 'compass-k8-.*?-virtual-daily-.*?' block-level: 'NODE' wrappers: diff --git a/jjb/container4nfv/arm64/compass-deploy.sh b/jjb/container4nfv/arm64/compass-deploy.sh index 3c599279e..a22af09ba 100755 --- a/jjb/container4nfv/arm64/compass-deploy.sh +++ b/jjb/container4nfv/arm64/compass-deploy.sh @@ -5,9 +5,14 @@ cd compass4nfv export ADAPTER_OS_PATTERN='(?i)CentOS-7.*arm.*' export OS_VERSION="centos7" -export KUBERNETES_VERSION="v1.7.3" -export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml" -export NETWORK="deploy/conf/vm_environment/network.yml" -export VIRT_NUMBER=2 VIRT_CPUS=2 VIRT_MEM=4096 VIRT_DISK=50G +export KUBERNETES_VERSION="v1.7.5" +if [[ "$NODE_NAME" =~ "-virtual" ]]; then + export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml" + export NETWORK="deploy/conf/vm_environment/network.yml" + export VIRT_NUMBER=2 VIRT_CPUS=4 VIRT_MEM=8192 VIRT_DISK=50G +else + export DHA="deploy/conf/hardware_environment/huawei-pod8/k8-nosdn-nofeature-noha.yml" + export NETWORK="deploy/conf/hardware_environment/huawei-pod8/network.yml" +fi ./deploy.sh diff --git a/jjb/container4nfv/arm64/yardstick-arm64.sh b/jjb/container4nfv/arm64/yardstick-arm64.sh new file mode 100755 index 000000000..26c6fdcfe --- /dev/null +++ b/jjb/container4nfv/arm64/yardstick-arm64.sh @@ -0,0 +1,93 @@ +#!/bin/bash +set -e + +sshpass -p root ssh root@10.1.0.50 \ + "mkdir -p /etc/yardstick; rm -rf /etc/yardstick/admin.conf" + + +sshpass -p root ssh root@10.1.0.50 \ + kubectl config set-cluster yardstick --server=127.0.0.1:8080 --insecure-skip-tls-verify=true --kubeconfig=/etc/yardstick/admin.conf +sshpass -p root ssh root@10.1.0.50 \ + kubectl config set-context yardstick --cluster=yardstick --kubeconfig=/etc/yardstick/admin.conf +sshpass -p root ssh root@10.1.0.50 \ + kubectl config use-context yardstick --kubeconfig=/etc/yardstick/admin.conf + + + +if [ ! -n "$redirect" ]; then + redirect="/dev/stdout" +fi + +if [ ! -n "$DOCKER_TAG" ]; then + DOCKER_TAG='latest' +fi + +if [ ! -n "$NODE_NAME" ]; then + NODE_NAME='arm-virutal03' +fi + +if [ ! -n "$DEPLOY_SCENARIO" ]; then + DEPLOY_SCENARIO='k8-nosdn-lb-noha_daily' +fi + +if [ ! -n "$YARDSTICK_DB_BACKEND" ]; then + YARDSTICK_DB_BACKEND='-i 104.197.68.199:8086' +fi + +# Pull the image with correct tag +DOCKER_REPO='opnfv/yardstick' +if [ "$(uname -m)" = 'aarch64' ]; then + DOCKER_REPO="${DOCKER_REPO}_$(uname -m)" +fi +echo "Yardstick: Pulling image ${DOCKER_REPO}:${DOCKER_TAG}" +sshpass -p root ssh root@10.1.0.50 \ + docker pull ${DOCKER_REPO}:$DOCKER_TAG >$redirect + +if [ ! -n "$BRANCH" ]; then + BRANCH=master +fi + +opts="--name=yardstick --privileged=true --net=host -d -it " +envs="-e YARDSTICK_BRANCH=${BRANCH} -e BRANCH=${BRANCH} \ + -e NODE_NAME=${NODE_NAME} \ + -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" +rc_file_vol="-v /etc/yardstick/admin.conf:/etc/yardstick/admin.conf" +cacert_file_vol="" +map_log_dir="" +sshkey="" +YARDSTICK_SCENARIO_SUITE_NAME="opnfv_k8-nosdn-lb-noha_daily.yaml" + +# map log directory +branch=${BRANCH##*/} +#branch="master" +dir_result="${HOME}/opnfv/yardstick/results/${branch}" +mkdir -p ${dir_result} +sudo rm -rf ${dir_result}/* +map_log_dir="-v ${dir_result}:/tmp/yardstick" + +# Run docker +cmd="docker rm -f yardstick || true" +sshpass -p root ssh root@10.1.0.50 \ + ${cmd} + +cmd="sudo docker run ${opts} ${envs} ${rc_file_vol} ${cacert_file_vol} ${map_log_dir} ${sshkey} ${DOCKER_REPO}:${DOCKER_TAG} /bin/bash" +echo "Yardstick: Running docker cmd: ${cmd}" +sshpass -p root ssh root@10.1.0.50 \ + ${cmd} + + +cmd='sudo docker exec yardstick sed -i.bak "/# execute tests/i\sed -i.bak \"s/openretriever\\\/yardstick/openretriever\\\/yardstick_aarch64/g\" \ + $\{YARDSTICK_REPO_DIR\}/tests/opnfv/test_cases/opnfv_yardstick_tc080.yaml" /usr/local/bin/exec_tests.sh' +sshpass -p root ssh root@10.1.0.50 \ + ${cmd} + +echo "Yardstick: run tests: ${YARDSTICK_SCENARIO_SUITE_NAME}" +cmd="sudo docker exec yardstick exec_tests.sh ${YARDSTICK_DB_BACKEND} ${YARDSTICK_SCENARIO_SUITE_NAME}" +sshpass -p root ssh root@10.1.0.50 \ + ${cmd} + +cmd="docker rm -f yardstick" +sshpass -p root ssh root@10.1.0.50 \ + ${cmd} + +echo "Yardstick: done!" diff --git a/jjb/container4nfv/container4nfv-arm64.yml b/jjb/container4nfv/container4nfv-arm64.yml index 16a10eac1..324624387 100644 --- a/jjb/container4nfv/container4nfv-arm64.yml +++ b/jjb/container4nfv/container4nfv-arm64.yml @@ -9,15 +9,20 @@ disabled: false - 'k8-sriov-nofeature-noha': disabled: false + pod: + - virtual: + slave-label: arm-packet01 + - baremetal: + slave-label: compass-baremetal-arm jobs: - - 'container4nfv-{scenario}-virtual-daily-master' + - 'container4nfv-{scenario}-{pod}-daily-master' - job-template: - name: 'container4nfv-{scenario}-virtual-daily-master' + name: 'container4nfv-{scenario}-{pod}-daily-master' disabled: '{obj:disabled}' concurrent: false - node: arm-packet01 + node: '{slave-label}' scm: - git: @@ -28,11 +33,11 @@ wipe-workspace: true triggers: - - 'trigger-{scenario}-virtual' + - 'trigger-{scenario}-{pod}' wrappers: - timeout: - timeout: 120 + timeout: 150 fail: true builders: @@ -40,6 +45,11 @@ !include-raw: arm64/compass-build.sh - shell: !include-raw: arm64/compass-deploy.sh + - trigger-builds: + - project: yardstick-arm64-compass-arm-virtual03-daily-master + current-parameters: false + same-node: true + block: false - trigger: @@ -50,3 +60,12 @@ name: 'trigger-k8-sriov-nofeature-noha-virtual' triggers: - timed: '0 16 * * *' + +- trigger: + name: 'trigger-k8-multus-nofeature-noha-baremetal' + triggers: + - timed: '0 16 * * *' +- trigger: + name: 'trigger-k8-sriov-nofeature-noha-baremetal' + triggers: + - timed: '0 20 * * *' diff --git a/jjb/container4nfv/yardstick-arm64.yml b/jjb/container4nfv/yardstick-arm64.yml new file mode 100644 index 000000000..03c68738a --- /dev/null +++ b/jjb/container4nfv/yardstick-arm64.yml @@ -0,0 +1,121 @@ +--- +################################### +# job configuration for yardstick +################################### +- project: + name: yardstick-arm64 + + project: 'yardstick' + + # ------------------------------- + # BRANCH ANCHORS + # ------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + docker-tag: 'latest' + # ------------------------------- + # POD, INSTALLER, AND BRANCH MAPPING + # ------------------------------- + # Installers using labels + # CI PODs + # This section should only contain the installers + # that have been switched using labels for slaves + # ------------------------------- + pod: + # apex CI PODs + - arm-virtual03: + slave-label: arm-packet01 + installer: compass + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + # ------------------------------- + testsuite: + - 'daily' + + jobs: + - 'yardstick-arm64-{installer}-{pod}-{testsuite}-{stream}' + +################################ +# job templates +################################ +- job-template: + name: 'yardstick-arm64-{installer}-{pod}-{testsuite}-{stream}' + disabled: false + + concurrent: true + + properties: + - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - timeout: + timeout: 60 + abort: true + + triggers: + - '{auto-trigger-name}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{installer}-defaults' + - 'yardstick-params-{slave-label}' + - string: + name: DEPLOY_SCENARIO + default: 'k8-nosdn-lb-noha_daily' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' + - string: + name: YARDSTICK_SCENARIO_SUITE_NAME + default: opnfv_${{DEPLOY_SCENARIO}}_{testsuite}.yaml + description: 'Path to test scenario suite' + - string: + name: CI_DEBUG + default: 'false' + description: "Show debut output information" + + scm: + - git-scm + + builders: + - description-setter: + description: "POD: $NODE_NAME" + - 'yardstick-arm64' + + publishers: + - email: + recipients: trevor.tao@arm.com yibo.cai@arm.com + - email-jenkins-admins-on-failure + +######################## +# builder macros +######################## +- builder: + name: yardstick-arm64 + builders: + - shell: + !include-raw: arm64/yardstick-arm64.sh + +######################## +# parameter macros +######################## + + +- parameter: + name: 'yardstick-params-arm-packet01' + parameters: + - string: + name: YARDSTICK_DB_BACKEND + default: '-i 104.197.68.199:8086' + description: 'Arguments to use in order to choose the backend DB' diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index 6f4643ac8..453451f52 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -32,24 +32,24 @@ # CI PODs # ------------------------------- pod: - # - baremetal: - # slave-label: daisy-baremetal - # <<: *master - - virtual: - slave-label: daisy-virtual - <<: *master - baremetal: slave-label: daisy-baremetal - <<: *euphrates + <<: *master - virtual: slave-label: daisy-virtual - <<: *euphrates + <<: *master + # - baremetal: + # slave-label: daisy-baremetal + # <<: *euphrates + # - virtual: + # slave-label: daisy-virtual + # <<: *euphrates # ------------------------------- # None-CI PODs # ------------------------------- - - baremetal: - slave-label: zte-pod3 - <<: *master + # - baremetal: + # slave-label: zte-pod3 + # <<: *master - zte-pod9: slave-label: zte-pod9 <<: *master @@ -117,9 +117,7 @@ - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' - - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + - 'testapi-parameter' - 'daisy-project-parameter': gs-pathname: '{gs-pathname}' @@ -192,8 +190,10 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'daisy.*-deploy-({pod})?-daily-.*' - 'daisy-kolla-build-.*' + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' block-level: 'NODE' parameters: @@ -203,9 +203,7 @@ - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' - - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' + - 'testapi-parameter' - 'daisy-project-parameter': gs-pathname: '{gs-pathname}' - string: @@ -337,7 +335,7 @@ - trigger: name: 'daisy-os-nosdn-ovs_dpdk-noha-zte-pod9-daily-master-trigger' triggers: - - timed: '0 10 * * *' + - timed: '0 10,20 * * *' # Basic HA Scenarios - trigger: name: 'daisy-os-nosdn-nofeature-ha-zte-pod9-daily-master-trigger' diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index 006153f6f..c4edd7738 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -215,6 +215,13 @@ enabled: true max-total: 6 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + block-level: 'NODE' scm: - git-scm diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index d8f43c9b2..8c04ffd53 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -19,7 +19,7 @@ installer: - 'apex' - 'fuel' - # - 'joid' + - 'daisy' arch: - 'x86_64' @@ -29,19 +29,17 @@ - 'sample' - 'congress' - task: - - verify: - auto-trigger-name: 'doctor-verify' - exclude: - installer: 'apex' arch: 'aarch64' + - installer: 'daisy' + arch: 'aarch64' jobs: - 'doctor-verify-{stream}' - 'doctor-verify-unit-test-{stream}' - - 'doctor-{task}-{inspector}-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + - 'doctor-verify-{inspector}-{stream}' + - 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' - job-template: name: 'doctor-verify-{stream}' @@ -76,6 +74,7 @@ file-paths: - compare-type: ANT pattern: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -88,11 +87,14 @@ projects: - name: 'doctor-verify-unit-test-{stream}' predefined-parameters: | + PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE kill-phase-on: FAILURE + current-parameters: true + git-revision: true - job-template: name: 'doctor-verify-unit-test-{stream}' @@ -114,16 +116,17 @@ - 'doctor-verify-unit-test-publishers-macro' - job-template: - name: 'doctor-{task}-{inspector}-{stream}' + name: 'doctor-verify-{inspector}-{stream}' disabled: '{obj:disabled}' project-type: 'multijob' scm: - git-scm-gerrit triggers: - - '{auto-trigger-name}': + - 'doctor-verify': project: '{project}' branch: '{branch}' files: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -134,38 +137,60 @@ name: 'doctor-verify-apex-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-apex-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-apex-{inspector}-x86_64-{stream}' predefined-parameters: | + PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE kill-phase-on: FAILURE + current-parameters: true + git-revision: true - multijob: name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-x86_64-{stream}' predefined-parameters: | + PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE kill-phase-on: FAILURE + current-parameters: true + git-revision: true - multijob: name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-aarch64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-aarch64-{stream}' + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + kill-phase-on: FAILURE + current-parameters: true + git-revision: true + - multijob: + name: 'doctor-verify-daisy-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-verify-daisy-{inspector}-x86_64-{stream}' predefined-parameters: | + PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE kill-phase-on: FAILURE + current-parameters: true + git-revision: true - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + name: 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' disabled: '{obj:disabled}' node: 'doctor-{installer}-{arch}' wrappers: diff --git a/jjb/dovetail/dovetail-ci-jobs.yml b/jjb/dovetail/dovetail-ci-jobs.yml index 99867c327..5e5b6e18f 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yml +++ b/jjb/dovetail/dovetail-ci-jobs.yml @@ -20,18 +20,12 @@ dovetail-branch: '{stream}' gs-pathname: '' docker-tag: 'latest' - danube: &danube - stream: danube - branch: 'stable/{stream}' - dovetail-branch: master - gs-pathname: '/{stream}' - docker-tag: 'cvp.0.9.0' euphrates: &euphrates stream: euphrates branch: 'stable/{stream}' dovetail-branch: master gs-pathname: '/{stream}' - docker-tag: 'cvp.0.9.0' + docker-tag: 'latest' # ---------------------------------- # POD, PLATFORM, AND BRANCH MAPPING @@ -60,16 +54,6 @@ slave-label: fuel-baremetal SUT: fuel auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - virtual: - slave-label: fuel-virtual - SUT: fuel - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - baremetal: - slave-label: fuel-baremetal - SUT: fuel - auto-trigger-name: 'daily-trigger-disabled' <<: *euphrates - virtual: slave-label: fuel-virtual @@ -91,16 +75,6 @@ slave-label: compass-baremetal SUT: compass auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - virtual: - slave-label: compass-virtual - SUT: compass - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - baremetal: - slave-label: compass-baremetal - SUT: compass - auto-trigger-name: 'daily-trigger-disabled' <<: *euphrates - virtual: slave-label: compass-virtual @@ -125,16 +99,6 @@ auto-trigger-name: 'daily-trigger-disabled' <<: *master - virtual: - slave-label: apex-virtual-danube - SUT: apex - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - baremetal: - slave-label: apex-baremetal-danube - SUT: apex - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - virtual: slave-label: apex-virtual-master SUT: apex auto-trigger-name: 'daily-trigger-disabled' @@ -159,16 +123,6 @@ slave-label: armband-baremetal SUT: fuel auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - armband-virtual: - slave-label: armband-virtual - SUT: fuel - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - armband-baremetal: - slave-label: armband-baremetal - SUT: fuel - auto-trigger-name: 'daily-trigger-disabled' <<: *euphrates - armband-virtual: slave-label: armband-virtual @@ -202,26 +156,6 @@ slave-label: zte-pod1 SUT: fuel auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - zte-pod3: - slave-label: zte-pod3 - SUT: fuel - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - huawei-pod4: - slave-label: huawei-pod4 - SUT: apex - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - huawei-pod7: - slave-label: huawei-pod7 - SUT: compass - auto-trigger-name: 'daily-trigger-disabled' - <<: *danube - - zte-pod1: - slave-label: zte-pod1 - SUT: fuel - auto-trigger-name: 'daily-trigger-disabled' <<: *euphrates - zte-pod3: slave-label: zte-pod3 @@ -241,6 +175,7 @@ # ------------------------------- testsuite: + - 'default' - 'proposed_tests' jobs: diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index e50242bd6..ec879e325 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -199,6 +199,27 @@ if [[ ! -f ${cirros_image} ]]; then fi sudo cp ${cirros_image} ${DOVETAIL_CONFIG} +# snaps_smoke test case needs to download this image first before running +ubuntu14_image=${image_path}/ubuntu-14.04-server-cloudimg-amd64-disk1.img +if [[ ! -f ${ubuntu14_image} ]]; then + echo "Download image ubuntu-14.04-server-cloudimg-amd64-disk1.img ..." + wget -q -nc https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img -P ${image_path} +fi +sudo cp ${ubuntu14_image} ${DOVETAIL_CONFIG} + +# cloudify_ims test case needs to download these 2 images first before running +cloudify_image=${image_path}/cloudify-manager-premium-4.0.1.qcow2 +if [[ ! -f ${cloudify_image} ]]; then + echo "Download image cloudify-manager-premium-4.0.1.qcow2 ..." + wget -q -nc http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 -P ${image_path} +fi +sudo cp ${cloudify_image} ${DOVETAIL_CONFIG} +trusty_image=${image_path}/trusty-server-cloudimg-amd64-disk1.img +if [[ ! -f ${trusty_image} ]]; then + echo "Download image trusty-server-cloudimg-amd64-disk1.img ..." + wget -q -nc http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img -P ${image_path} +fi +sudo cp ${trusty_image} ${DOVETAIL_CONFIG} opts="--privileged=true -id" @@ -238,20 +259,6 @@ if [ $(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | wc -l) == 0 ]; then exit 1 fi -if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then - source_cmd="source ${OPENRC}" - get_public_url_cmd="openstack --insecure endpoint list --service keystone --interface public | sed -n 4p | awk '{print \$14}'" - public_url=$(sudo docker exec "$container_id" /bin/bash -c "${source_cmd} && ${get_public_url_cmd}") - sed -i 's#OS_AUTH_URL=.*#OS_AUTH_URL='"${public_url}"'#g' ${OPENRC} - sed -i 's/internal/public/g' ${OPENRC} - if [[ ${public_url} =~ 'v2' ]]; then - sed -i "s/OS_IDENTITY_API_VERSION=3/OS_IDENTITY_API_VERSION=2.0/g" ${OPENRC} - sed -i '/OS_PROJECT_DOMAIN_NAME/d' ${OPENRC} - sed -i '/OS_USER_DOMAIN_NAME/d' ${OPENRC} - fi - cat ${OPENRC} -fi - # Modify tempest_conf.yaml file tempest_conf_file=${DOVETAIL_CONFIG}/tempest_conf.yaml if [[ ${INSTALLER_TYPE} == 'compass' || ${INSTALLER_TYPE} == 'apex' ]]; then @@ -275,10 +282,13 @@ cp_tempest_cmd="docker cp ${DOVETAIL_CONFIG}/tempest_conf.yaml $container_id:/ho echo "exec command: ${cp_tempest_cmd}" $cp_tempest_cmd -list_cmd="dovetail list ${TESTSUITE}" -run_cmd="dovetail run --testsuite ${TESTSUITE} -d" -echo "Container exec command: ${list_cmd}" -docker exec $container_id ${list_cmd} +if [[ ${TESTSUITE} == 'default' ]]; then + testsuite='' +else + testsuite="--testsuite ${TESTSUITE}" +fi + +run_cmd="dovetail run ${testsuite} -d" echo "Container exec command: ${run_cmd}" docker exec $container_id ${run_cmd} diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 1cb29d091..a198751a8 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -63,6 +63,8 @@ auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-nofeature-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' + - 'os-ovn-nofeature-ha': + auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-ovs-ha': @@ -101,6 +103,14 @@ stream: master - scenario: os-odl_l2-bgpvpn-ha stream: euphrates + - scenario: os-ovn-nofeature-noha + stream: danube + - scenario: os-ovn-nofeature-noha + stream: euphrates + - scenario: os-ovn-nofeature-ha + stream: danube + - scenario: os-ovn-nofeature-ha + stream: euphrates jobs: - 'fuel-{scenario}-{pod}-daily-{stream}' @@ -161,33 +171,73 @@ DEPLOY_SCENARIO={scenario} same-node: true block: true + - trigger-builds: + - project: 'functest-fuel-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - trigger-builds: + - project: 'yardstick-fuel-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + # 1.here the stream means the SUT stream, dovetail stream is defined in its own job + # 2.testsuite proposed_tests here is for new test cases planning to add into OVP + # 3.run proposed_tests on Monday, Wednesday and Friday against ha scenario + # 4.testsuite default here is for the test cases already added into OVP + # 5.run default testsuite on Tuesday against ha scenario + # 6.not used for release criteria or compliance, only to debug the dovetail tool bugs - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + MON: true + WED: true + FRI: true + use-build-time: true steps: - trigger-builds: - - project: 'functest-fuel-{pod}-daily-{stream}' + - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} - same-node: true block: true + same-node: true block-thresholds: build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - conditional-step: - condition-kind: not - condition-operand: - condition-kind: regex-match - regex: 'danube' - label: '{stream}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: '.*-ha' + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + TUES: true + use-build-time: true steps: - trigger-builds: - - project: 'yardstick-fuel-{pod}-daily-{stream}' + - project: 'dovetail-fuel-{pod}-default-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -197,21 +247,6 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.here the stream means the SUT stream, dovetail stream is defined in its own job - # 2.only debug testsuite here(refstack, ha, vping, ipv6, tempest, bgpvpn) - # 3.not used for release criteria or compliance, only to debug the dovetail tool bugs - # 4.ZTE pod1, os-nosdn-nofeature-ha and os-odl-bgpvpn-ha, run against danube - - trigger-builds: - - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - conditional-step: condition-kind: not condition-operand: @@ -357,6 +392,10 @@ triggers: - timed: '' # '5 8 * * *' - trigger: + name: 'fuel-os-ovn-nofeature-ha-baremetal-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-ha-baremetal-daily-master-trigger' triggers: - timed: '' # '5 17 * * *' @@ -412,11 +451,11 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-baremetal-daily-euphrates-trigger' triggers: - - timed: '0 20 * * *' + - timed: '' # '0 20 * * *' - trigger: name: 'fuel-os-odl-nofeature-ha-baremetal-daily-euphrates-trigger' triggers: - - timed: '0 2 * * *' + - timed: '' # '0 2 * * *' - trigger: name: 'fuel-os-onos-sfc-ha-baremetal-daily-euphrates-trigger' triggers: @@ -432,7 +471,7 @@ - trigger: name: 'fuel-os-nosdn-ovs-ha-baremetal-daily-euphrates-trigger' triggers: - - timed: '0 20 * * *' + - timed: '' # '0 20 * * *' - trigger: name: 'fuel-os-nosdn-kvm_ovs_dpdk-ha-baremetal-daily-euphrates-trigger' triggers: @@ -459,10 +498,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-ovn-nofeature-noha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-baremetal-daily-euphrates-trigger' triggers: - timed: '' @@ -498,6 +533,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-ha-virtual-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-ha-virtual-daily-master-trigger' triggers: - timed: '' @@ -589,11 +628,11 @@ - trigger: name: 'fuel-os-nosdn-nofeature-noha-virtual-daily-euphrates-trigger' triggers: - - timed: '0 13 * * *' + - timed: '' # '0 13 * * *' - trigger: name: 'fuel-os-odl-nofeature-noha-virtual-daily-euphrates-trigger' triggers: - - timed: '0 18 * * *' + - timed: '' # '0 18 * * *' - trigger: name: 'fuel-os-onos-sfc-noha-virtual-daily-euphrates-trigger' triggers: @@ -603,17 +642,13 @@ triggers: - timed: '' # '0 23 * * *' - trigger: - name: 'fuel-os-ovn-nofeature-noha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-euphrates-trigger' triggers: - timed: '' # '30 6 * * *' - trigger: name: 'fuel-os-nosdn-ovs-noha-virtual-daily-euphrates-trigger' triggers: - - timed: '0 9 * * *' + - timed: '' # '0 9 * * *' - trigger: name: 'fuel-os-nosdn-kvm_ovs_dpdk-noha-virtual-daily-euphrates-trigger' triggers: @@ -642,6 +677,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-ha-zte-pod1-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod1-daily-master-trigger' triggers: - timed: '' diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index b5db2ed37..432bbbb8b 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -4,6 +4,10 @@ set -e set +u set +o pipefail +CI_LOOP=${CI_LOOP:-daily} +TEST_DB_URL=http://testresults.opnfv.org/test/api/v1/results +ENERGY_RECORDER_API_URL=http://energy.opnfv.fr/resources + check_os_deployment() { FUNCTEST_IMAGE=opnfv/functest-healthcheck:${DOCKER_TAG} echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." @@ -145,7 +149,8 @@ test -f ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG} && custom_params=$(ca envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ - -e BUILD_TAG=${BUILD_TAG} -e DEPLOY_TYPE=${DEPLOY_TYPE}" + -e BUILD_TAG=${BUILD_TAG} -e DEPLOY_TYPE=${DEPLOY_TYPE} -e CI_LOOP=${CI_LOOP} \ + -e TEST_DB_URL=${TEST_DB_URL} -e ENERGY_RECORDER_API_URL=${ENERGY_RECORDER_API_URL}" ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index 84bdac6db..a58e0bf3f 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -197,9 +197,9 @@ - 'suite': job-timeout: 60 - 'daily': - job-timeout: 300 + job-timeout: 360 - 'arm-daily': - job-timeout: 300 + job-timeout: 360 jobs: - 'functest-{installer}-{pod}-{testsuite}-{stream}' @@ -249,32 +249,13 @@ builders: - description-setter: description: "Built on $NODE_NAME" - - conditional-step: - condition-kind: regex-match - regex: "os-*" - label: 'DEPLOY_SCENARIO' - steps: - - 'functest-{testsuite}-builder' - - conditional-step: - condition-kind: regex-match - regex: "k8-*" - label: 'DEPLOY_SCENARIO' - steps: - - 'functest-k8-builder' + - 'functest-{testsuite}-builder' ######################## # parameter macros ######################## - parameter: - name: functest-k8-parameter - parameters: - - string: - name: FUNCTEST_MODE - default: 'daily' - description: "Daily suite name to run on K8 deployment" - -- parameter: name: functest-daily-parameter parameters: - string: @@ -399,31 +380,28 @@ - 'functest-exit' - builder: - name: functest-k8-builder - builders: - - 'functest-cleanup' - - 'functest-daily-k8' - - 'functest-store-results' - - 'functest-exit' - -- builder: - name: functest-daily-k8 - builders: - # yamllint disable rule:indentation - - shell: - !include-raw: - - ../../utils/fetch_k8_conf.sh - - ./functest-k8.sh - -- builder: name: functest-daily builders: # yamllint disable rule:indentation - - shell: - !include-raw: - - ./functest-env-presetup.sh - - ../../utils/fetch_os_creds.sh - - ./functest-alpine.sh + - conditional-step: + condition-kind: regex-match + regex: "os-.*" + label: '$DEPLOY_SCENARIO' + steps: + - shell: + !include-raw: + - ./functest-env-presetup.sh + - ../../utils/fetch_os_creds.sh + - ./functest-alpine.sh + - conditional-step: + condition-kind: regex-match + regex: "k8-.*" + label: '$DEPLOY_SCENARIO' + steps: + - shell: + !include-raw: + - ../../utils/fetch_k8_conf.sh + - ./functest-k8.sh # yamllint enable rule:indentation - builder: diff --git a/jjb/releng/functest-docker.yml b/jjb/functest/functest-docker.yml index 584dc302c..909c7b079 100644 --- a/jjb/releng/functest-docker.yml +++ b/jjb/functest/functest-docker.yml @@ -72,7 +72,7 @@ disabled: '{obj:disabled}' parameters: - - job-parameters: + - functest-job-parameters: project: '{project}' branch: '{branch}' slave_label: 'opnfv-build-ubuntu' @@ -168,7 +168,7 @@ name: 'functest-{image}-docker-build-{arch_tag}-{stream}' disabled: '{obj:disabled}' parameters: - - job-parameters: + - functest-job-parameters: project: '{project}' branch: '{branch}' slave_label: '{slave_label}' @@ -205,6 +205,11 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + disabled: '{obj:disabled}' @@ -219,13 +224,13 @@ esac sudo manifest-tool push from-args \ --platforms linux/amd64,linux/arm64 \ - --template opnfv/functest-{image}:ARCH-$tag \ - --target opnfv/functest-{image}:$tag + --template $REPO/functest-{image}:ARCH-$tag \ + --target $REPO/functest-{image}:$tag exit $? # parameter macro - parameter: - name: job-parameters + name: functest-job-parameters parameters: - project-parameter: project: '{project}' @@ -262,6 +267,10 @@ name: PROJECT default: "{project}" description: "Project name used to enable job conditions" + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" # publisher macros - publisher: @@ -281,3 +290,4 @@ jalausuch@suse.com morgan.richomme@orange.com cedric.ollivier@orange.com feng.xiaowei@zte.com.cn juha.kosonen@nokia.com wangwulin@huawei.com + valentin.boucher@kontron.com diff --git a/jjb/functest/functest-env-presetup.sh b/jjb/functest/functest-env-presetup.sh index cd15d7194..b7d33e760 100755 --- a/jjb/functest/functest-env-presetup.sh +++ b/jjb/functest/functest-env-presetup.sh @@ -38,7 +38,7 @@ elif [[ ${INSTALLER_TYPE} == 'daisy' ]]; then installer_mac=$(sudo virsh domiflist daisy | grep vnet | \ grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") - export INSTALLER_IP=$(/usr/sbin/arp -e -i $bridge_name | grep ${installer_mac} | awk {'print $1'}) + export INSTALLER_IP=$(/usr/sbin/arp -e -i $bridge_name | grep ${installer_mac} | head -n 1 | awk {'print $1'}) echo "Installer ip is ${INSTALLER_IP}" else diff --git a/jjb/functest/functest-k8.sh b/jjb/functest/functest-k8.sh index 6df5c53b0..3f08e4a0b 100755 --- a/jjb/functest/functest-k8.sh +++ b/jjb/functest/functest-k8.sh @@ -11,13 +11,16 @@ rc_file=${HOME}/k8.creds sudo rm -rf $rc_file if [[ ${INSTALLER_TYPE} == 'compass' ]]; then - admin_conf_file_vol = "-v ${HOME}/admin.conf:/root/.kube/config" + admin_conf_file_vol="-v ${HOME}/admin.conf:/root/.kube/config" echo "export KUBECONFIG=/root/.kube/config" >> $rc_file echo "export KUBERNETES_PROVIDER=local" >> $rc_file - KUBE_MASTER_URL = $(cat ${HOME}/admin.conf|grep server| awk '{print $2}') + KUBE_MASTER_URL=$(cat ${HOME}/admin.conf|grep server| awk '{print $2}') echo "export KUBE_MASTER_URL=$KUBE_MASTER_URL" >> $rc_file - KUBE_MASTER_IP = $(echo $KUBE_MASTER_URL|awk -F'https://|:[0-9]+' '$0=$2') + KUBE_MASTER_IP=$(echo $KUBE_MASTER_URL|awk -F'https://|:[0-9]+' '$0=$2') echo "export KUBE_MASTER_IP=$KUBE_MASTER_IP" >> $rc_file +elif [[ ${INSTALLER_TYPE} == 'joid' && ${BRANCH} == 'master' ]]; then + admin_conf_file_vol="-v ${HOME}/joid_config/config:/root/joid_config/config" + rc_file=${HOME}/joid_config/k8config else echo "Not supported by other installers yet" exit 1 @@ -44,9 +47,10 @@ envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} \ DOCKER_TAG=`[[ ${BRANCH##*/} == "master" ]] && echo "latest" || echo ${BRANCH##*/}` FUNCTEST_IMAGE=opnfv/functest-kubernetes:${DOCKER_TAG} +echo "Pulling Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null cmd_opt="run_tests -r -t all" -cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" +cmd="docker run --rm --privileged=true ${envs} ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest k8s test cases, CMD: ${cmd}" eval ${cmd} ret_value=$? diff --git a/jjb/functest/functest-kubernetes-project-jobs.yml b/jjb/functest/functest-kubernetes-project-jobs.yml new file mode 100644 index 000000000..3eba5d177 --- /dev/null +++ b/jjb/functest/functest-kubernetes-project-jobs.yml @@ -0,0 +1,257 @@ +--- +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: functest-kubernetes-project-jobs + + project: 'functest-kubernetes' + + jobs: + - 'functest-kubernetes-verify-{stream}' + - 'functest-kubernetes-verify-{phase}-{stream}' + - 'functest-kubernetes-docs-upload-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - euphrates: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false + + phase: + - 'unit-tests-and-docs': + slave-label: 'opnfv-build-ubuntu' + - 'build-x86_64': + slave-label: 'opnfv-build-ubuntu' + - 'build-aarch64': + slave-label: 'opnfv-build-ubuntu-arm' + +- job-template: + name: 'functest-kubernetes-verify-{stream}' + + disabled: '{obj:disabled}' + + project-type: 'multijob' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - 'functest-kubernetes-verify-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'functest-kubernetes-build-and-unittest' + execution-type: PARALLEL + projects: + - name: 'functest-kubernetes-verify-unit-tests-and-docs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'functest-kubernetes-verify-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=x86_64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'functest-kubernetes-verify-build-aarch64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=aarch64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + +- job-template: + name: 'functest-kubernetes-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 30 + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{slave-label}-defaults' + + scm: + - git-scm-gerrit + + builders: + - 'functest-kubernetes-verify-{phase}-builders-macro' + + publishers: + - 'functest-kubernetes-verify-{phase}-publishers-macro' + +- job-template: + name: 'functest-kubernetes-docs-upload-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - 'functest-kubernetes-docs-upload-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - functest-kubernetes-upload-doc-artifact + +################################ +# job triggers +################################ +- trigger: + name: 'functest-kubernetes-verify-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + +- trigger: + name: 'functest-kubernetes-docs-upload-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' +################################ +# job builders +################################ + +- builder: + name: 'functest-kubernetes-verify-unit-tests-and-docs-builders-macro' + builders: + - shell: | + cd $WORKSPACE && tox + +- builder: + name: 'functest-kubernetes-verify-build-x86_64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'functest-kubernetes-verify-build-aarch64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'functest-kubernetes-upload-doc-artifact' + builders: + - shell: | + cd $WORKSPACE && tox -edocs + wget -O - https://git.opnfv.org/releng/plain/utils/upload-artifact.sh | bash -s "api/_build" "docs" +################################ +# job publishers +################################ +- publisher: + name: 'functest-kubernetes-verify-unit-tests-and-docs-publishers-macro' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + - email-jenkins-admins-on-failure + +- publisher: + name: 'functest-kubernetes-verify-build-x86_64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure + +- publisher: + name: 'functest-kubernetes-verify-build-aarch64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure diff --git a/jjb/functest/xtesting-docker.yml b/jjb/functest/xtesting-docker.yml new file mode 100644 index 000000000..dc26344e3 --- /dev/null +++ b/jjb/functest/xtesting-docker.yml @@ -0,0 +1,221 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: xtesting-docker + + project: functest-xtesting + + stream: + - master: + branch: '{stream}' + disabled: false + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + - 'arm64': + slave_label: 'opnfv-build-ubuntu-arm' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + GERRIT_REFNAME=$GERRIT_REFNAME + DOCKERFILE=$DOCKERFILE + kill-phase-on: FAILURE + abort-all-jobs: false + + manifest-job-settings: &manifest-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: + GERRIT_REFNAME=$GERRIT_REFNAME + kill-phase-on: FAILURE + abort-all-jobs: false + + # yamllint enable rule:key-duplicates + jobs: + - "xtesting-docker-{stream}" + - "xtesting-docker-build-{arch_tag}-{stream}" + - "xtesting-docker-manifest-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'xtesting-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - xtesting-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/30 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' + + builders: + - multijob: + name: 'build xtesting images' + execution-type: PARALLEL + projects: + - name: 'xtesting-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'xtesting-docker-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish xtesting manifests' + execution-type: PARALLEL + projects: + - name: 'xtesting-docker-manifest-{stream}' + <<: *manifest-job-settings + + publishers: + - 'xtesting-amd64-recipients' + - 'xtesting-arm64-recipients' + +- job-template: + name: 'xtesting-docker-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - xtesting-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + scm: + - git-scm + builders: + - shell: | + #!/bin/bash -ex + sudo arch={arch_tag} bash ./build.sh + exit $? + +- job-template: + name: 'xtesting-docker-manifest-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'opnfv-build-ubuntu' + description: 'Slave label on Jenkins' + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + + + disabled: '{obj:disabled}' + + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template $REPO/xtesting:ARCH-$tag \ + --target $REPO/xtesting:$tag + exit $? + +- parameter: + name: xtesting-job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: GERRIT_REFNAME + default: "" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" + - string: + name: DOCKERFILE + default: "Dockerfile" + description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "{arch_tag}" + description: "If set, this value will be added to the docker image tag as a prefix" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + +# publisher macros +- publisher: + name: 'xtesting-arm64-recipients' + publishers: + - email: + recipients: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + delia.popescu@enea.com + +- publisher: + name: 'xtesting-amd64-recipients' + publishers: + - email: + recipients: > + jalausuch@suse.com morgan.richomme@orange.com + cedric.ollivier@orange.com feng.xiaowei@zte.com.cn + juha.kosonen@nokia.com wangwulin@huawei.com + valentin.boucher@kontron.com diff --git a/jjb/functest/xtesting-project-jobs.yml b/jjb/functest/xtesting-project-jobs.yml new file mode 100644 index 000000000..bf08775b6 --- /dev/null +++ b/jjb/functest/xtesting-project-jobs.yml @@ -0,0 +1,257 @@ +--- +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: xtesting-project-jobs + + project: 'functest-xtesting' + + jobs: + - 'xtesting-verify-{stream}' + - 'xtesting-verify-{phase}-{stream}' + - 'xtesting-docs-upload-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - euphrates: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false + + phase: + - 'unit-tests-and-docs': + slave-label: 'opnfv-build-ubuntu' + - 'build-x86_64': + slave-label: 'opnfv-build-ubuntu' + - 'build-aarch64': + slave-label: 'opnfv-build-ubuntu-arm' + +- job-template: + name: 'xtesting-verify-{stream}' + + disabled: '{obj:disabled}' + + project-type: 'multijob' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - 'xtesting-verify-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'xtesting-build-and-unittest' + execution-type: PARALLEL + projects: + - name: 'xtesting-verify-unit-tests-and-docs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'xtesting-verify-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=x86_64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'xtesting-verify-build-aarch64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=aarch64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + +- job-template: + name: 'xtesting-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 30 + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{slave-label}-defaults' + + scm: + - git-scm-gerrit + + builders: + - 'xtesting-verify-{phase}-builders-macro' + + publishers: + - 'xtesting-verify-{phase}-publishers-macro' + +- job-template: + name: 'xtesting-docs-upload-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - 'xtesting-docs-upload-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - xtesting-upload-doc-artifact + +################################ +# job triggers +################################ +- trigger: + name: 'xtesting-verify-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + +- trigger: + name: 'xtesting-docs-upload-triggers-macro' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' +################################ +# job builders +################################ + +- builder: + name: 'xtesting-verify-unit-tests-and-docs-builders-macro' + builders: + - shell: | + cd $WORKSPACE && tox + +- builder: + name: 'xtesting-verify-build-x86_64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'xtesting-verify-build-aarch64-builders-macro' + builders: + - shell: | + echo "Not activated!" + +- builder: + name: 'xtesting-upload-doc-artifact' + builders: + - shell: | + cd $WORKSPACE && tox -edocs + wget -O - https://git.opnfv.org/releng/plain/utils/upload-artifact.sh | bash -s "api/_build" "docs" +################################ +# job publishers +################################ +- publisher: + name: 'xtesting-verify-unit-tests-and-docs-publishers-macro' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + - email-jenkins-admins-on-failure + +- publisher: + name: 'xtesting-verify-build-x86_64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure + +- publisher: + name: 'xtesting-verify-build-aarch64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure diff --git a/jjb/global/installer-report.sh b/jjb/global/installer-report.sh index e2fcfd6f6..6cd83f1f1 100755 --- a/jjb/global/installer-report.sh +++ b/jjb/global/installer-report.sh @@ -8,18 +8,39 @@ ############################################################################## source $WORKSPACE/installer_track.sh -echo """ - INSTALLER: $INSTALLER - INSTALLER_VERSION: $INSTALLER_VERSION - JOB_NAME: $JOB_NAME - BUILD_ID: $BUILD_ID - SENARIO: $DEPLOY_SCENARIO - UPSTREAM_JOB_NAME: $UPSTREAM_JOB_NAME: - UPSTREAM_BUILD_ID: $UPSTREAM_BUILD_ID - PROVISION_RESULT: $PROVISION_RESULT - TIMESTAMP_START: $TIMESTAMP_START - TIMESTAMP_END: `date '+%Y-%m-%d %H:%M:%S.%3N'` - POD_NAME: $NODE_NAME -""" -# TODO call TestAPI to report installer provisoin result when API is ready +gen_content() +{ + cat <<EOF +{ + "installer": "$INSTALLER", + "version": "$INSTALLER_VERSION", + "pod_name": "$NODE_NAME", + "job_name": "$JOB_NAME", + "build_id": "$BUILD_ID", + "scenario": "$DEPLOY_SCENARIO", + "upstream_job_name": "$UPSTREAM_JOB_NAME", + "upstream_build_id":"$UPSTREAM_BUILD_ID", + "criteria": "$PROVISION_RESULT", + "start_date": "$TIMESTAMP_START", + "stop_date": "$TIMESTAMP_END", + "details":"" +} +EOF +} + +echo "Installer: $INSTALLER provision result: $PROVISION_RESULT" +echo $(gen_content) + +set -o xtrace +curl -H "Content-Type: application/json" -X POST -v -d "$(gen_content)" \ + $TESTAPI_URL/deployresults || true + +# INFO +# postbuildscript plugin shall always return the original job running status, +# for the result returned from postbuildscript affects the CI pipeline. +if [ "$PROVISION_RESULT" == "PASS" ]; then + exit 0 +else + exit 1 +fi diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index bd9f97d3b..205802fab 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -40,6 +40,30 @@ default: 'refs/heads/{branch}' description: "Default refspec needed for manually triggering." +- parameter: + name: testapi-parameter + parameters: + - string: + name: TESTAPI_URL + default: 'http://testresults.opnfv.org/test/api/v1' + description: "Default TestAPI URL, currently using v1" + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + description: "OPNFV deployment scenario" + - string: + name: INSTALLER_VERSION + default: 'master' + description: "Installer release version" + - string: + name: UPSTREAM_JOB_NAME + default: '' + description: "Parent job name in Jenkins" + - string: + name: UPSTREAM_BUILD_ID + default: '' + description: "Parent job build_id in Jenkins" + - property: name: logrotate-default properties: @@ -318,9 +342,11 @@ - shell: | #!/bin/bash if [[ -s violation.log ]]; then - echo "Reporting lint result..." + cat violation.log + echo "Reporting lint result...." + set -x msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset." - sed -i -e '1s/^//$msg\n\n/' violation.log + sed -i -e "1s#^#${msg}\n\n#" violation.log cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE" ssh -p 29418 gerrit.opnfv.org "$cmd" @@ -348,7 +374,7 @@ builders: - shell: | #!/bin/bash - # Install python package + # Install python package sudo pip install "flake8==2.6.2" echo "Checking python code..." @@ -617,7 +643,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - hongbo.tianhongbo@huawei.com + georg.kunz@ericsson.com - publisher: name: 'email-dpacc-ptl' @@ -825,7 +851,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - jack.morgan@intel.com + zhang.jun3g@zte.com.cn - publisher: name: 'email-pharos-tools-ptl' <<: *email_pharos_ptl_defaults @@ -901,7 +927,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - ManuelBuilmbuil@suse.com + mbuil@suse.com - publisher: name: 'email-snaps-ptl' @@ -958,15 +984,18 @@ script-only-if-succeeded: true builders: - shell: | - echo "export PROVISION_RESULT=SUCCEED" >> $WORKSPACE/installer_track.sh - echo "export INSTALLER={installer}" >> $WORKSPACE/installer_track.sh + echo "export PROVISION_RESULT=PASS" >> $WORKSPACE/installer_track.sh + echo "export INSTALLER=$INSTALLER_TYPE" >> $WORKSPACE/installer_track.sh + echo "export TIMESTAMP_END="\'`date '+%Y-%m-%d %H:%M:%S.%3N'`\' >> $WORKSPACE/installer_track.sh - shell: - !include-raw-escape: installer-report.sh + !include-raw: installer-report.sh - postbuildscript: + script-only-if-succeeded: false script-only-if-failed: true builders: - shell: | echo "export PROVISION_RESULT=FAIL" >> $WORKSPACE/installer_track.sh - echo "export INSTALLER={installer}" >> $WORKSPACE/installer_track.sh + echo "export INSTALLER=$INSTALLER_TYPE" >> $WORKSPACE/installer_track.sh + echo "export TIMESTAMP_END="\'`date '+%Y-%m-%d %H:%M:%S.%3N'`\' >> $WORKSPACE/installer_track.sh - shell: - !include-raw-escape: installer-report.sh + !include-raw: installer-report.sh diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 4902dcfc5..6171a6abd 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -18,13 +18,6 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' - - node: - name: SLAVE_NAME - description: 'Slave name on Jenkins' - allowed-slaves: - - lf-pod1 - default-slaves: - - lf-pod1 - parameter: name: 'apex-baremetal-euphrates-defaults' @@ -40,13 +33,6 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' - - node: - name: SLAVE_NAME - description: 'Slave name on Jenkins' - allowed-slaves: - - lf-pod1 - default-slaves: - - lf-pod1 - parameter: name: 'apex-baremetal-danube-defaults' @@ -62,13 +48,7 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' - - node: - name: SLAVE_NAME - description: 'Slave name on Jenkins' - allowed-slaves: - - lf-pod1 - default-slaves: - - lf-pod1 + - parameter: name: 'apex-virtual-master-defaults' @@ -84,15 +64,6 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' - - node: - name: SLAVE_NAME - description: 'Slave name on Jenkins' - allowed-slaves: - - lf-virtual2 - - lf-virtual3 - default-slaves: - - lf-virtual2 - - lf-virtual3 - parameter: name: 'apex-virtual-euphrates-defaults' @@ -108,15 +79,6 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' - - node: - name: SLAVE_NAME - description: 'Slave name on Jenkins' - allowed-slaves: - - lf-virtual2 - - lf-virtual3 - default-slaves: - - lf-virtual2 - - lf-virtual3 - parameter: name: 'apex-virtual-danube-defaults' @@ -132,13 +94,6 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' - - node: - name: SLAVE_NAME - description: 'Slave name on Jenkins' - allowed-slaves: - - lf-pod3 - default-slaves: - - lf-pod3 - parameter: name: 'lf-pod1-defaults' @@ -1028,6 +983,7 @@ - 'doctor-fuel-x86_64' - 'doctor-fuel-aarch64' - 'doctor-joid-x86_64' + - 'doctor-daisy-x86_64' - 'not-specified' default-slaves: - 'not-specified' diff --git a/jjb/opera/opera-daily-jobs.yml b/jjb/opera/opera-daily-jobs.yml index 5d7526bba..f0841a34b 100644 --- a/jjb/opera/opera-daily-jobs.yml +++ b/jjb/opera/opera-daily-jobs.yml @@ -11,7 +11,7 @@ stream: master branch: '{stream}' gs-pathname: '' - disabled: false + disabled: true ##################################### # pod definitions diff --git a/jjb/opera/opera-verify-jobs.yml b/jjb/opera/opera-verify-jobs.yml index ad93d4653..97c9a43ec 100644 --- a/jjb/opera/opera-verify-jobs.yml +++ b/jjb/opera/opera-verify-jobs.yml @@ -11,7 +11,7 @@ - master: branch: '{stream}' gs-pathname: '' - disabled: false + disabled: true ##################################### # patch verification phases diff --git a/jjb/pharos/check-jinja2.yml b/jjb/pharos/check-jinja2.yml index 7251d3f61..3ef2f04e5 100644 --- a/jjb/pharos/check-jinja2.yml +++ b/jjb/pharos/check-jinja2.yml @@ -68,11 +68,11 @@ - compare-type: ANT pattern: '**/*.yaml' - compare-type: ANT - pattern: 'config/installers/**/*.j2' + pattern: 'config/utils/*.py' - compare-type: ANT - pattern: 'check-jinja2.sh' + pattern: 'config/installers/**/*.j2' - compare-type: ANT - pattern: 'check-schema.sh' + pattern: 'check-*.sh' skip-vote: successful: false failed: false diff --git a/jjb/pharos/pharos.yml b/jjb/pharos/pharos.yml index acf14885e..c1bb1ba0e 100644 --- a/jjb/pharos/pharos.yml +++ b/jjb/pharos/pharos.yml @@ -11,7 +11,8 @@ jobs: - '{project}-verify-basic' - 'backup-pharos-dashboard' - - 'deploy-pharos-dashboard' + - 'deploy-pharos-dashboard': + disabled: true - job-template: name: 'backup-pharos-dashboard' @@ -52,7 +53,16 @@ - 'pharos-dashboard-defaults' scm: - - git-scm-gerrit + - git: + choosing-strategy: 'gerrit' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$BRANCH' + timeout: 15 + credentials-id: '$SSH_CREDENTIAL_ID' + url: '$GIT_BASE' + skip-tag: true + wipe-workspace: false triggers: - gerrit: @@ -71,6 +81,8 @@ builders: - shell: | cp $HOME/config.env $WORKSPACE/dashboard + cp $HOME/rsa.pub $WORKSPACE/dashboard + cp $HOME/rsa.pem $WORKSPACE/dashboard cd $WORKSPACE/dashboard sudo docker-compose build sudo docker-compose up -d diff --git a/jjb/qtip/qtip-experimental-jobs.yml b/jjb/qtip/qtip-experimental-jobs.yml index 103069fb7..b130912f3 100644 --- a/jjb/qtip/qtip-experimental-jobs.yml +++ b/jjb/qtip/qtip-experimental-jobs.yml @@ -13,10 +13,6 @@ branch: '{stream}' gs-pathname: '' disabled: false - - euphrates: - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false pod: - zte-virtual6: diff --git a/jjb/qtip/qtip-validate-jobs.yml b/jjb/qtip/qtip-validate-jobs.yml index d9d3e72e6..5c0bb881b 100644 --- a/jjb/qtip/qtip-validate-jobs.yml +++ b/jjb/qtip/qtip-validate-jobs.yml @@ -27,22 +27,17 @@ - compute: installer: apex pod: zte-virtual5 + sut: node <<: *master - storage: installer: apex pod: zte-virtual5 + sut: '' <<: *master - compute: - installer: apex - pod: zte-virtual5 - <<: *euphrates - - storage: - installer: apex - pod: zte-virtual5 - <<: *euphrates - - compute: installer: fuel pod: zte-virtual6 + sut: vnf <<: *master # ------------------------------- @@ -84,6 +79,10 @@ - string: name: OPNFV_RELEASE default: '{stream}' + - string: + name: SUT + description: "System Under Test" + default: '{sut}' scm: - git-scm diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml index 7f142241c..5fbb37ca2 100644 --- a/jjb/releng/compass4nfv-docker.yml +++ b/jjb/releng/compass4nfv-docker.yml @@ -27,6 +27,8 @@ - 'db' - 'deck' - 'tasks-base' + - 'tasks-k8s' + - 'tasks-osa' # settings for jobs run in multijob phases build-job-settings: &build-job-settings @@ -95,6 +97,10 @@ <<: *build-job-settings - name: 'compass-tasks-build-amd64-{stream}' <<: *build-job-settings + - name: 'compass-tasks-k8s-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-tasks-osa-build-amd64-{stream}' + <<: *build-job-settings publishers: - 'compass-amd64-recipients' diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index 28e5b7ce1..5c8cafbe3 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -111,7 +111,7 @@ fi echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" echo "--------------------------------------------------------" echo -cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH +cmd="docker build --pull=true --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH $ARCH_BUILD_ARG -f $DOCKERFILE $DOCKER_PATH" diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 317efc807..55cbc5f1d 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -64,6 +64,11 @@ dockerdir: 'docker/barometer-ves' <<: *master <<: *other-receivers + - 'barometer-snmp': + project: 'barometer' + dockerdir: 'docker/barometer-snmp' + <<: *master + <<: *other-receivers - 'bottlenecks': project: 'bottlenecks' <<: *master @@ -86,6 +91,11 @@ dockerfile: 'docker/Dockerfile.local' <<: *master <<: *other-receivers + - 'qtip-nettest': + project: 'qtip' + dockerdir: 'contrib/nettest' + <<: *master + <<: *other-receivers - 'storperf-master': project: 'storperf' dockerdir: 'docker/storperf-master' diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index e9e929de6..0ac552082 100644 --- a/jjb/releng/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -117,7 +117,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' + project-pattern: 'functest|functest-kubernetes|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index 6f77cd250..fb3bab42b 100644 --- a/jjb/releng/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml @@ -7,6 +7,8 @@ - 'prune-docker-images' - 'archive-repositories' - 'check-status-of-slaves' + - 'ansible-build-server' + - 'generate-artifacts-index-pages' ######################## # job templates @@ -23,7 +25,10 @@ name: SLAVE_NAME description: Slaves to prune docker images default-slaves: - - arm-build2 + - arm-build3 + - arm-build4 + - arm-build5 + - arm-build6 - ericsson-build3 - ericsson-build4 - lf-build2 @@ -33,12 +38,14 @@ builders: - description-setter: description: "Built on $NODE_NAME" + # yamllint disable rule:line-length - shell: | #!/bin/bash - (docker ps -q; docker ps -aq) | sort | uniq -u | xargs --no-run-if-empty docker rm docker images -f dangling=true -q | xargs --no-run-if-empty docker rmi + + # yamllint enable rule:line-length triggers: - timed: '@midnight' @@ -82,22 +89,121 @@ name: 'check-status-of-slaves' disabled: false - concurrent: true parameters: - node: name: SLAVE_NAME - description: We don't want workspace wiped. so I just threw the script on the master + description: 'script lives on master node' default-slaves: - - master + - lf-build1 allowed-multiselect: false ignore-offline-nodes: true + - project-parameter: + project: releng + branch: master + + scm: + - git-scm triggers: - timed: '@midnight' builders: - shell: | - cd /opt/jenkins-ci/slavemonitor - bash slave-monitor-0.1.sh | sort + cd $WORKSPACE/utils/ + bash slave-monitor-0.1.sh + +- job-template: + name: 'ansible-build-server' + + project-type: freestyle + + disabled: false + concurrent: true + + parameters: + - node: + name: SLAVE_NAME + description: Build Servers + default-slaves: + - lf-build1 + - lf-build2 + allowed-multiselect: true + ignore-offline-nodes: true + - project-parameter: + project: releng + branch: master + + scm: + - git-scm + + triggers: + - timed: '@midnight' + + builders: + - install-ansible + - run-ansible-build-server-playbook + + +- builder: + name: install-ansible + builders: + - shell: | + # Install ansible here + if [ -f /etc/centos-release ] \ + || [ -f /etc/redhat-release ] \ + || [ -f /etc/system-release ]; then + sudo yum -y install ansible + fi + if [ -f /etc/debian_version ] \ + || grep -qi ubuntu /etc/lsb-release \ + || grep -qi ubuntu /etc/os-release; then + sudo apt-get -y install ansible + fi + +- builder: + name: run-ansible-build-server-playbook + builders: + - shell: | + # run playbook + sudo ansible-playbook -i \ + $WORKSPACE/utils/build-server-ansible/inventory.ini \ + $WORKSPACE/utils/build-server-ansible/main.yml + + +- job-template: + name: 'generate-artifacts-index-pages' + + project-type: freestyle + + disabled: false + + concurrent: false + + parameters: + - node: + name: SLAVE_NAME + description: Build Servers + default-slaves: + - lf-build2 + - project-parameter: + project: releng + branch: master + + scm: + - git-scm + + triggers: + - timed: '@hourly' + + builders: + - generate-artifact-html + + +- builder: + name: generate-artifact-html + builders: + - shell: | + cd $WORKSPACE/utils/ + ./artifacts.opnfv.org.sh diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index ef99f5abb..6fb6804bc 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -121,6 +121,4 @@ builders: - shell: | source /opt/virtualenv/jenkins-job-builder/bin/activate - cd /opt/jenkins-ci/releng - git pull jenkins-jobs update -r --delete-old jjb/ diff --git a/jjb/releng/automate.yml b/jjb/releng/testresults-automate.yml index 6bd952a1d..01fbe0c6d 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/testresults-automate.yml @@ -1,6 +1,6 @@ --- - project: - name: utils-automate + name: testresults-automate stream: - master: branch: '{stream}' @@ -17,7 +17,6 @@ jobs: - '{module}-automate-{stream}' - '{module}-automate-{phase}-{stream}' - - '{module}-verify-{stream}' project: 'releng-testresults' @@ -53,53 +52,6 @@ send-to-individuals: true - job-template: - name: '{module}-verify-{stream}' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: '{module}/**' - - builders: - - shell: | - cd {module}/ - tox - if [ -e *.xml ];then - cp *.xml $WORKSPACE - fi - - publishers: - - publish-coverage - - email-jenkins-admins-on-failure - -- job-template: name: '{module}-automate-{stream}' project-type: multijob @@ -135,21 +87,10 @@ fail: true triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: '{module}/**' + - gerrit-trigger-change-merged: + project: '**' + branch: '{branch}' + files: '{module}/**' builders: - description-setter: diff --git a/jjb/releng/testresults-verify.yml b/jjb/releng/testresults-verify.yml new file mode 100644 index 000000000..958833c0f --- /dev/null +++ b/jjb/releng/testresults-verify.yml @@ -0,0 +1,78 @@ +--- +- project: + name: testresults-verify + stream: + - master: + branch: '{stream}' + + module: + - 'testapi' + - 'reporting' + + jobs: + - '{module}-verify-{stream}' + - '{module}-client-verify-{stream}' + + project: 'releng-testresults' + +- job-template: + name: '{module}-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit-trigger-patchset-created: + server: 'gerrit.opnfv.org' + project: '**' + branch: '{branch}' + files: '{module}/**' + + builders: + - shell: | + cd {module}/ + tox + if [ -e *.xml ];then + cp *.xml $WORKSPACE + fi + + publishers: + - publish-coverage + - email-jenkins-admins-on-failure + +- job-template: + name: '{module}-client-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit-trigger-patchset-created: + server: 'gerrit.opnfv.org' + project: '**' + branch: '{branch}' + files: '{module}/{module}-client/**' + + builders: + - shell: | + cd {module}/{module}-client + tox + if [ -e *.xml ];then + cp *.xml $WORKSPACE + fi + + publishers: + - publish-coverage + - email-jenkins-admins-on-failure diff --git a/jjb/vswitchperf/vswitchperf.yml b/jjb/vswitchperf/vswitchperf.yml index 16ceb2e52..7fd875bcb 100644 --- a/jjb/vswitchperf/vswitchperf.yml +++ b/jjb/vswitchperf/vswitchperf.yml @@ -26,6 +26,16 @@ disabled: '{obj:disabled}' + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'vswitchperf-verify-.*' + - 'vswitchperf-merge-.*' + - 'vswitchperf-daily-.*' + block-level: 'NODE' + parameters: - project-parameter: project: '{project}' @@ -45,10 +55,8 @@ cd src make clobber make MORE_MAKE_FLAGS="-j 10" - # run basic sanity test - make sanity cd ../ci - scl enable python33 "source ~/vsperfenv/bin/activate ; ./build-vsperf.sh daily" + scl enable rh-python34 "source ~/vsperfenv/bin/activate ; ./build-vsperf.sh daily" - job-template: name: 'vswitchperf-verify-{stream}' @@ -66,6 +74,7 @@ blocking-jobs: - 'vswitchperf-verify-.*' - 'vswitchperf-merge-.*' + - 'vswitchperf-daily-.*' block-level: 'NODE' parameters: @@ -80,8 +89,10 @@ name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: + - intel-pod12 - ericsson-build4 default-slaves: + - intel-pod12 - ericsson-build4 scm: @@ -117,8 +128,6 @@ cd src make clobber make MORE_MAKE_FLAGS="-j 5" - # run basic sanity test - make sanity cd ../ci ./build-vsperf.sh verify @@ -138,6 +147,7 @@ blocking-jobs: - 'vswitchperf-verify-.*' - 'vswitchperf-merge-.*' + - 'vswitchperf-daily-.*' block-level: 'NODE' parameters: @@ -152,8 +162,10 @@ name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: + - intel-pod12 - ericsson-build4 default-slaves: + - intel-pod12 - ericsson-build4 scm: diff --git a/jjb/xci/bifrost-periodic-jobs.yml b/jjb/xci/bifrost-periodic-jobs.yml index 0bca26aa5..fbe2e205a 100644 --- a/jjb/xci/bifrost-periodic-jobs.yml +++ b/jjb/xci/bifrost-periodic-jobs.yml @@ -123,7 +123,7 @@ name: SLAVE_LABEL default: '{slave-label}' - string: - name: XCI_LOOP + name: CI_LOOP default: 'periodic' wrappers: diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml index e4c2d0e5b..2fb7c2167 100644 --- a/jjb/xci/bifrost-verify-jobs.yml +++ b/jjb/xci/bifrost-verify-jobs.yml @@ -71,7 +71,14 @@ use-build-blocker: true blocking-jobs: - 'xci-verify-{distro}-.*' + - 'xci-.*-{distro}-merge-.*' - '.*-bifrost-verify.*-{type}' + - throttle: + max-per-node: 2 + max-total: 10 + categories: + - xci-verify-virtual + option: category block-level: 'NODE' @@ -122,7 +129,7 @@ name: SLAVE_LABEL default: 'xci-virtual' - string: - name: XCI_LOOP + name: CI_LOOP default: 'verify' scm: @@ -147,7 +154,7 @@ wrappers: - fix-workspace-permissions - build-timeout: - timeout: 90 + timeout: 180 publishers: # yamllint disable rule:line-length @@ -155,6 +162,13 @@ recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com # yamllint enable rule:line-length - email-jenkins-admins-on-failure + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh + # ------------------------------- # trigger macros # ------------------------------- diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh index 198f2e1c7..9219d0020 100755 --- a/jjb/xci/bifrost-verify.sh +++ b/jjb/xci/bifrost-verify.sh @@ -21,9 +21,14 @@ git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD cd $WORKSPACE/releng-xci cat > bifrost_test.sh<<EOF +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + cd ~/bifrost # provision 3 VMs; xcimaster, controller, and compute -./scripts/bifrost-provision.sh +./scripts/bifrost-provision.sh | ts sudo -H -E virsh list EOF @@ -41,6 +46,6 @@ export XCI_UPDATE_CLEAN_VM_OS=true ./xci/scripts/vm/start-new-vm.sh $VM_DISTRO -rsync -a -e "ssh -F $HOME/.ssh/xci-vm-config" $WORKSPACE/ ${VM_DISTRO}_xci_vm:~/bifrost +rsync -a -e "ssh -F $HOME/.ssh/${VM_DISTRO}-xci-vm-config" $WORKSPACE/ ${VM_DISTRO}_xci_vm:~/bifrost -ssh -F $HOME/.ssh/xci-vm-config ${VM_DISTRO}_xci_vm "cd ~/bifrost/releng-xci && ./bifrost_test.sh" +ssh -F $HOME/.ssh/${VM_DISTRO}-xci-vm-config ${VM_DISTRO}_xci_vm "cd ~/bifrost/releng-xci && ./bifrost_test.sh" diff --git a/jjb/xci/osa-periodic-jobs.yml b/jjb/xci/osa-periodic-jobs.yml index 64daa696b..6c2d165a4 100644 --- a/jjb/xci/osa-periodic-jobs.yml +++ b/jjb/xci/osa-periodic-jobs.yml @@ -216,6 +216,7 @@ export FUNCTEST_MODE=$FUNCTEST_MODE export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME export XCI_FLAVOR=$XCI_FLAVOR + export CORE_OPENSTACK_INSTALL=true export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ export INSTALLER_TYPE=$INSTALLER_TYPE @@ -234,7 +235,7 @@ - shell: | #!/bin/bash - ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" + ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" - builder: @@ -247,8 +248,8 @@ - shell: | #!/bin/bash - sudo virsh destroy ${DISTRO}_xci_vm - sudo virsh undefine ${DISTRO}_xci_vm + sudo virsh destroy ${DISTRO}_xci_vm || true + sudo virsh undefine ${DISTRO}_xci_vm || true # this will be enabled once the xci is prepared # - builder: diff --git a/jjb/xci/xci-cleanup.sh b/jjb/xci/xci-cleanup.sh index 45b4ec273..51a863da2 100755 --- a/jjb/xci/xci-cleanup.sh +++ b/jjb/xci/xci-cleanup.sh @@ -14,11 +14,15 @@ # what you are doing. #---------------------------------------------------------------------- +# Need to cover macros with and without parameters +VM_NAME=$DISTRO +VM_NAME+=_xci_vm + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then echo "Skipping the deployment!" exit 0 fi -sudo virsh destroy ${DISTRO}_xci_vm -sudo virsh undefine ${DISTRO}_xci_vm +sudo virsh destroy $VM_NAME || true +sudo virsh undefine $VM_NAME || true diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml index a95374933..11db4e168 100644 --- a/jjb/xci/xci-daily-jobs.yml +++ b/jjb/xci/xci-daily-jobs.yml @@ -113,7 +113,7 @@ name: SLAVE_LABEL default: '{slave-label}' - string: - name: XCI_LOOP + name: CI_LOOP default: 'daily' triggers: @@ -131,7 +131,7 @@ predefined-parameters: | DEPLOY_SCENARIO=$DEPLOY_SCENARIO XCI_FLAVOR=$XCI_FLAVOR - XCI_LOOP=$XCI_LOOP + CI_LOOP=$CI_LOOP same-node: true block: true - trigger-builds: @@ -140,7 +140,7 @@ predefined-parameters: | DEPLOY_SCENARIO=$DEPLOY_SCENARIO XCI_FLAVOR=$XCI_FLAVOR - XCI_LOOP=$XCI_LOOP + CI_LOOP=$CI_LOOP same-node: true block: true block-thresholds: @@ -154,6 +154,12 @@ recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com # yamllint enable rule:line-length - email-jenkins-admins-on-failure + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh - job-template: name: 'xci-{phase}-{pod}-{distro}-daily-{stream}' @@ -214,7 +220,7 @@ name: SLAVE_LABEL default: '{slave-label}' - string: - name: XCI_LOOP + name: CI_LOOP default: 'daily' - string: name: INSTALLER_TYPE diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh index 211d282d3..71cf96d8d 100755 --- a/jjb/xci/xci-deploy.sh +++ b/jjb/xci/xci-deploy.sh @@ -15,14 +15,14 @@ cd $WORKSPACE/xci # for daily jobs, we want to use working versions # for periodic jobs, we will use whatever is set in the job, probably master -if [[ "$XCI_LOOP" == "daily" ]]; then +if [[ "$CI_LOOP" == "daily" ]]; then # source pinned-vars to get releng version source ./config/pinned-versions # checkout the version git checkout -q $OPNFV_RELENG_VERSION echo "Info: Using $OPNFV_RELENG_VERSION" -elif [[ "$XCI_LOOP" == "periodic" ]]; then +elif [[ "$CI_LOOP" == "periodic" ]]; then echo "Info: Using $OPNFV_RELENG_VERSION" fi @@ -31,7 +31,7 @@ fi # to take this into account while deploying anyways # clone openstack-ansible # stable/ocata already use pinned versions so this is only valid for master -if [[ "$XCI_LOOP" == "periodic" && "$OPENSTACK_OSA_VERSION" == "master" ]]; then +if [[ "$CI_LOOP" == "periodic" && "$OPENSTACK_OSA_VERSION" == "master" ]]; then cd $WORKSPACE # get the url to openstack-ansible git source ./config/env-vars diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml index 351fe22bb..8a3a7cfbc 100644 --- a/jjb/xci/xci-merge-jobs.yml +++ b/jjb/xci/xci-merge-jobs.yml @@ -2,6 +2,7 @@ - project: name: 'opnfv-xci-merge' + project: releng-xci # ------------------------------- # branches # ------------------------------- @@ -19,34 +20,35 @@ - opensuse: disabled: false # ------------------------------- + # scenarios + # ------------------------------- + scenario: + - 'os-odl-sfc': + project: sfc + # ------------------------------- # type # ------------------------------- type: - virtual # ------------------------------- - # patch verification phases + # postmerge promotion phases # ------------------------------- phase: - 'deploy' - 'healthcheck' - - 'promote' - # ------------------------------- - # scenarios - # ------------------------------- - scenario: - - 'os-odl-sfc': - project: sfc # ------------------------------- # jobs # ------------------------------- jobs: - - 'xci-{scenario}-{type}-{distro}-merge-{stream}' - - 'xci-{phase}-{type}-{distro}-merge-{stream}' + - 'xci-{scenario}-{type}-merge-{stream}' + - 'xci-{scenario}-{type}-promote-merge-{stream}' + - 'xci-merge-{distro}-{type}-{stream}' + - 'xci-merge-{distro}-{phase}-{type}-{stream}' # ------------------------------- # job templates # ------------------------------- - job-template: - name: 'xci-{scenario}-{type}-{distro}-merge-{stream}' + name: 'xci-{scenario}-{type}-merge-{stream}' project-type: multijob @@ -56,16 +58,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'xci-verify-{distro}-.*' - - 'bifrost-verify-{distro}-.*' - - 'bifrost-periodic-{distro}-.*' - - 'xci-osa-verify-{distro}-.*' - - 'xci-osa-periodic-{distro}-.*' - - 'xci-(os|k8s).*?-virtual-{distro}-.*' - block-level: 'NODE' wrappers: - ssh-agent-wrapper @@ -74,12 +66,7 @@ - fix-workspace-permissions scm: - - git: - credentials-id: '$SSH_CREDENTIAL_ID' - url: 'https://gerrit.opnfv.org/gerrit/releng-xci' - branches: - - 'origin/master' - timeout: 15 + - git-scm-gerrit triggers: - gerrit: @@ -103,14 +90,14 @@ parameters: - project-parameter: - project: '{obj:project}' + project: $GERRIT_PROJECT branch: '{branch}' - label: name: SLAVE_LABEL default: 'xci-virtual' - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + name: DISTRO + default: 'all' - string: name: CLEAN_DIB_IMAGES default: 'true' @@ -121,18 +108,139 @@ builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Node: $NODE_NAME" + - 'xci-merge-set-scenario-macro' + - multijob: + name: deploy and test + condition: SUCCESSFUL + projects: + - name: 'xci-merge-opensuse-{type}-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=opensuse + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - name: 'xci-merge-ubuntu-{type}-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=ubuntu + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - name: 'xci-merge-centos-{type}-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=centos + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - multijob: + name: promote + condition: SUCCESSFUL + projects: + - name: 'xci-{scenario}-{type}-promote-merge-{stream}' + current-parameters: true + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + +- job-template: + name: 'xci-merge-{distro}-{type}-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-verify-{distro}-.*' + - 'xci-merge-{distro}-.*' + - 'openstack-bifrost-verify-{distro}-.*' + - 'xci-osa-verify-{distro}-.*' + - 'xci-osa-periodic-{distro}-.*' + block-level: 'NODE' + - throttle: + max-per-node: 1 + max-total: 3 + categories: + - xci-verify-virtual + option: category + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + scm: + - git-scm-gerrit + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'xci-virtual' + - string: + name: DISTRO + default: '{distro}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + builders: + - 'xci-merge-set-scenario-macro' + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - description-setter: + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - multijob: name: deploy condition: SUCCESSFUL projects: - - name: 'xci-deploy-{type}-{distro}-merge-{stream}' + - name: 'xci-merge-{distro}-deploy-{type}-{stream}' current-parameters: true predefined-parameters: | - PROJECT=$PROJECT DISTRO={distro} - DEPLOY_SCENARIO={scenario} CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -144,25 +252,32 @@ name: healthcheck condition: SUCCESSFUL projects: - - name: 'xci-healthcheck-{type}-{distro}-merge-{stream}' + - name: 'xci-merge-{distro}-healthcheck-{type}-{stream}' current-parameters: true predefined-parameters: | - PROJECT=$PROJECT DISTRO={distro} - DEPLOY_SCENARIO={scenario} CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + FUNCTEST_MODE=tier + FUNCTEST_TIER=healthcheck node-parameters: true kill-phase-on: NEVER abort-all-job: true + publishers: + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh + - job-template: - name: 'xci-{phase}-{type}-{distro}-merge-{stream}' + name: 'xci-merge-{distro}-{phase}-{type}-{stream}' disabled: false @@ -182,14 +297,11 @@ parameters: - string: name: PROJECT - default: 'releng-xci' + default: $GERRIT_PROJECT - string: name: DISTRO default: 'ubuntu' - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-noha' - - string: name: FUNCTEST_MODE default: 'tier' - string: @@ -205,53 +317,90 @@ name: OPNFV_RELENG_DEV_PATH default: $WORKSPACE/ - string: - name: INSTALLER_TYPE - default: 'osa' - - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - ssh-agent-wrapper - build-timeout: timeout: 240 - fix-workspace-permissions scm: - - git: - credentials-id: '$SSH_CREDENTIAL_ID' - url: 'https://gerrit.opnfv.org/gerrit/releng-xci' - branches: - - 'origin/master' - timeout: 15 + - git-scm-gerrit builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - 'xci-merge-{phase}-macro' +- job-template: + name: 'xci-{scenario}-{type}-promote-merge-{stream}' + + disabled: false + + concurrent: false + + properties: + - logrotate-default + + parameters: + - string: + name: PROJECT + default: $GERRIT_PROJECT + - string: + name: DISTRO + default: 'all' + - string: + name: OPNFV_RELENG_DEV_PATH + default: $WORKSPACE/ + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + builders: + - description-setter: + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" + - 'xci-merge-promote-macro' + # ------------------------------- # builder macros # ------------------------------- - builder: - name: 'xci-merge-deploy-macro' + name: 'xci-merge-set-scenario-macro' builders: - shell: - !include-raw: ./xci-start-new-vm.sh - - shell: - !include-raw: ./xci-start-deployment.sh + !include-raw: ./xci-set-scenario.sh + +- builder: + name: 'xci-merge-deploy-macro' + builders: + - shell: | + #!/bin/bash + echo "Hello World" - builder: name: 'xci-merge-healthcheck-macro' builders: - - shell: - !include-raw: ./xci-run-functest.sh - - shell: - !include-raw: ./xci-cleanup.sh + - shell: | + #!/bin/bash + echo "Hello World" - builder: name: 'xci-merge-promote-macro' builders: - - shell: - !include-raw: ./xci-promote.sh + - shell: | + #!/bin/bash + echo "Hello World" diff --git a/jjb/xci/xci-promote.sh b/jjb/xci/xci-promote.sh index 98ad7ff4a..62818abee 100755 --- a/jjb/xci/xci-promote.sh +++ b/jjb/xci/xci-promote.sh @@ -14,6 +14,9 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the healthcheck if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the healthcheck!" diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh index b2421de32..1f616de1f 100755 --- a/jjb/xci/xci-run-functest.sh +++ b/jjb/xci/xci-run-functest.sh @@ -7,6 +7,9 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +set -o errexit +set -o nounset +set -o pipefail #---------------------------------------------------------------------- # This script is used by CI and executed by Jenkins jobs. @@ -14,12 +17,33 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the healthcheck if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the healthcheck!" exit 0 fi +# skip the healthcheck if the scenario is Kubernetes scenario +if [[ "$DEPLOY_SCENARIO" =~ k8 ]]; then + echo "Skipping the healthcheck!" + exit 0 +fi + +# if the scenario is external, we need to wipe WORKSPACE to place releng-xci there since +# the project where the scenario is coming from is cloned and the patch checked out to the +# xci/scenarios/$DEPLOY_SCENARIO to be synched on clean VM +# apart from that, we need releng-xci stuff in WORKSPACE for things to function correctly on Jenkins. +# if the change is coming to releng-xci, we don't need to do anything since the patch is checked +# out to the WORKSPACE anyways +if [[ $GERRIT_PROJECT != "releng-xci" ]]; then + cd $HOME && /bin/rm -rf $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + chmod -R go-rwx $WORKSPACE/xci/scripts/vm +fi + # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then @@ -27,4 +51,10 @@ if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENT exit 0 fi -ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci/xci && PATH=/home/devuser/.local/bin:$PATH ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/run-functest.yml" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "cd /root/releng-xci/xci/playbooks && ansible-playbook -i inventory prepare-functest.yml" +echo "Running functest" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "/root/run-functest.sh" +echo "Functest log" +echo "---------------------------------------------------------------------------------" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "cat /root/results/functest.log" +echo "---------------------------------------------------------------------------------" diff --git a/jjb/xci/xci-set-scenario.sh b/jjb/xci/xci-set-scenario.sh new file mode 100755 index 000000000..c602957a9 --- /dev/null +++ b/jjb/xci/xci-set-scenario.sh @@ -0,0 +1,210 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o pipefail + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# This function allows developers to specify the impacted scenario by adding +# the info about installer and scenario into the commit message or using +# the topic branch names. This results in either skipping the real verification +# totally or skipping the determining the installer and scenario programmatically. +# It is important to note that this feature is only available to generic scenarios +# and only single installer/scenario pair is allowed. +# The input in commit message should be placed at the end of the commit message body, +# before the signed-off and change-id lines. +# +# Pattern to be searched in Commit Message +# deploy-scenario:<scenario-name> +# installer-type:<installer-type> +# Examples: +# deploy-scenario:os-odl-nofeature +# installer-type:osa +# +# deploy-scenario:k8-nosdn-nofeature +# installer-type:kubespray +# +# Patterns to be searched in topic branch name +# skip-verify +# skip-deployment +# force-verify +function override_generic_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # process topic branch names + if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then + # skip the real verification + echo "Skipping verify!" + echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + exit 0 + elif [[ "$GERRIT_TOPIC" =~ 'force-verify' ]]; then + # Run the deployment with default installer and scenario when multiple things change + # and we want to force that. + echo "Recording the installer 'osa' and scenario 'os-nosdn-nofeature' for downstream jobs" + echo "Forcing CI verification of default scenario and installer!" + echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + exit 0 + fi + + # process commit message + if [[ "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "installer-type:" && "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "deploy-scenario:" ]]; then + INSTALLER_TYPE=$(echo $GERRIT_CHANGE_COMMIT_MESSAGE | awk '/installer-type:/' RS=" " | cut -d":" -f2) + DEPLOY_SCENARIO=$(echo $GERRIT_CHANGE_COMMIT_MESSAGE | awk '/deploy-scenario:/' RS=" " | cut -d":" -f2) + + if [[ -z "$INSTALLER_TYPE" || -z "$DEPLOY_SCENARIO" ]]; then + echo "Installer type or deploy scenario is not specified. Falling back to programmatically determining them." + else + echo "Recording the installer '$INSTALLER_TYPE' and scenario '$DEPLOY_SCENARIO' for downstream jobs" + echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties + exit 0 + fi + else + echo "Installer type or deploy scenario is not specified. Falling back to programmatically determining them." + fi +} + +# This function determines the impacted generic scenario by processing the +# change and using diff to see what changed. If changed files belong to a scenario +# its name gets recorded for deploying and testing the right scenario. +# +# Pattern to be searched in Changeset +# releng-xci/scenarios/<scenario>/<impacted files>: <scenario> +# releng-xci/xci/installer/osa/<impacted files>: os-nosdn-nofeature +# releng-xci/xci/installer/kubespray/<impacted files>: k8-nosdn-nofeature +# the rest: os-nosdn-nofeature +function determine_generic_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # get the changeset + cd $WORKSPACE + CHANGESET=$(git diff HEAD^..HEAD --name-only) + for CHANGED_FILE in $CHANGESET; do + case $CHANGED_FILE in + *k8-nosdn*|*kubespray*) + [[ ${DEPLOY_SCENARIO[@]} =~ "k8-nosdn-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='k8-nosdn-nofeature' + ;; + *os-odl*) + [[ ${DEPLOY_SCENARIO[@]} =~ "os-odl-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='os-odl-nofeature' + ;; + *os-nosdn*|*osa*) + [[ ${DEPLOY_SCENARIO[@]} =~ "os-nosdn-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='os-nosdn-nofeature' + ;; + *) + [[ ${DEPLOY_SCENARIO[@]} =~ "os-nosdn-nofeature" ]] || DEPLOY_SCENARIO[${#DEPLOY_SCENARIO[@]}]='os-nosdn-nofeature' + ;; + esac + done + + # extract releng-xci sha + RELENG_XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha which is same as releng-xci sha for generic scenarios + SCENARIO_SHA=$RELENG_XCI_SHA +} + +# This function determines the impacted external scenario by processing the Gerrit +# change and using diff to see what changed. If changed files belong to a scenario +# its name gets recorded for deploying and testing the right scenario. +# +# Pattern +# <project-repo>/scenarios/<scenario>/<impacted files>: <scenario> +function determine_external_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # remove the clone that is done via jenkins and place releng-xci there so the + # things continue functioning properly + cd $HOME && /bin/rm -rf $WORKSPACE + git clone -q https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + + # fix the permissions so ssh doesn't complain due to having world-readable keyfiles + chmod -R go-rwx $WORKSPACE/xci/scripts/vm + + # clone the project repo and fetch the patchset to process for further processing + git clone -q https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT $WORK_DIRECTORY/$GERRIT_PROJECT + cd $WORK_DIRECTORY/$GERRIT_PROJECT + git fetch -q https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT $GERRIT_REFSPEC && git checkout -q FETCH_HEAD + + # process the diff to find out what scenario(s) are impacted - there should only be 1 + DEPLOY_SCENARIO+=$(git diff HEAD^..HEAD --name-only | grep scenarios | awk -F '[/|/]' '{print $2}' | uniq) + + # extract releng-xci sha + RELENG_XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha + SCENARIO_SHA=$(cd $WORK_DIRECTORY/$GERRIT_PROJECT && git rev-parse HEAD) +} + +echo "Determining the impacted scenario" + +declare -a DEPLOY_SCENARIO + +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + +# this directory is where the temporary clones and files are created +# while extracting the impacted scenario +WORK_DIRECTORY=/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO +/bin/rm -rf $WORK_DIRECTORY && mkdir -p $WORK_DIRECTORY + +if [[ $GERRIT_PROJECT == "releng-xci" ]]; then + override_generic_scenario + determine_generic_scenario +else + determine_external_scenario +fi + +# ensure single scenario is impacted + if [[ $(IFS=$'\n' echo ${DEPLOY_SCENARIO[@]} | wc -w) != 1 ]]; then + echo "Change impacts multiple scenarios!" + echo "XCI doesn't support testing of changes that impact multiple scenarios currently." + echo "Please split your change into multiple different/dependent changes, each modifying single scenario." + exit 1 +fi + +# set the installer +case ${DEPLOY_SCENARIO[0]} in + os-*) + INSTALLER_TYPE=osa + ;; + k8-*) + INSTALLER_TYPE=kubespray + ;; + *) + echo "Unable to determine the installer. Exiting!" + exit 1 + ;; +esac + +# save the installer and scenario names into java properties file +# so they can be injected to downstream jobs via envInject +echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' and SHAs for downstream jobs" +echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties +echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties +echo "RELENG_XCI_SHA=$RELENG_XCI_SHA" >> $WORK_DIRECTORY/scenario.properties +echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties + +# skip scenario support check if the job is promotion job +if [[ "$JOB_NAME" =~ (os|k8) ]]; then + exit 0 +fi + +# skip the deployment if the scenario is not supported on this distro +OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml +if ! sed -n "/^- scenario: ${DEPLOY_SCENARIO[0]}$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then + echo "# SKIPPED: Scenario ${DEPLOY_SCENARIO[0]} is NOT supported on $DISTRO" + exit 0 +fi diff --git a/jjb/xci/xci-start-deployment.sh b/jjb/xci/xci-start-deployment.sh index e86f5b3a1..102ca41c3 100755 --- a/jjb/xci/xci-start-deployment.sh +++ b/jjb/xci/xci-start-deployment.sh @@ -14,12 +14,27 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the deployment!" exit 0 fi +# if the scenario is external, we need to wipe WORKSPACE to place releng-xci there since +# the project where the scenario is coming from is cloned and the patch checked out to the +# xci/scenarios/$DEPLOY_SCENARIO to be synched on clean VM +# apart from that, we need releng-xci stuff in WORKSPACE for things to function correctly on Jenkins. +# if the change is coming to releng-xci, we don't need to do anything since the patch is checked +# out to the WORKSPACE anyways +if [[ $GERRIT_PROJECT != "releng-xci" ]]; then + cd $HOME && /bin/rm -rf $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + chmod -R go-rwx $WORKSPACE/xci/scripts/vm +fi + # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then @@ -27,4 +42,4 @@ if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENT exit 0 fi -ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" diff --git a/jjb/xci/xci-start-new-vm.sh b/jjb/xci/xci-start-new-vm.sh index b72c33920..7874e6776 100755 --- a/jjb/xci/xci-start-new-vm.sh +++ b/jjb/xci/xci-start-new-vm.sh @@ -14,12 +14,27 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then echo "Skipping the deployment!" exit 0 fi +# if the scenario is external, we need to wipe WORKSPACE to place releng-xci there since +# the project where the scenario is coming from is cloned and the patch checked out to the +# xci/scenarios/$DEPLOY_SCENARIO to be synched on clean VM +# apart from that, we need releng-xci stuff in WORKSPACE for things to function correctly on Jenkins. +# if the change is coming to releng-xci, we don't need to do anything since the patch is checked +# out to the WORKSPACE anyways +if [[ $GERRIT_PROJECT != "releng-xci" ]]; then + cd $HOME && /bin/rm -rf $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE && cd $WORKSPACE + chmod -R go-rwx $WORKSPACE/xci/scripts/vm +fi + # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then @@ -35,11 +50,14 @@ cd $WORKSPACE # yourself. cat > xci_test.sh<<EOF #!/bin/bash +set -o pipefail export DISTRO=$DISTRO export DEPLOY_SCENARIO=$DEPLOY_SCENARIO export FUNCTEST_MODE=$FUNCTEST_MODE export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME export XCI_FLAVOR=$XCI_FLAVOR +export CORE_OPENSTACK_INSTALL=true +export BIFROST_USE_PREBUILT_IMAGES=true export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ export INSTALLER_TYPE=$INSTALLER_TYPE @@ -53,7 +71,7 @@ if [[ ! -z ${WORKSPACE+x} && $GERRIT_PROJECT != "releng-xci" ]]; then fi cd xci -./xci-deploy.sh +./xci-deploy.sh | ts EOF chmod a+x xci_test.sh diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 28bc317bd..d78dc82d9 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -21,10 +21,10 @@ notbuilt: false - centos: disabled: false - successful: false - failed: false - unstable: false - notbuilt: false + successful: true + failed: true + unstable: true + notbuilt: true - opensuse: disabled: false successful: false @@ -66,8 +66,8 @@ use-build-blocker: true blocking-jobs: - 'xci-verify-{distro}-.*' - - 'bifrost-verify-{distro}-.*' - - 'bifrost-periodic-{distro}-.*' + - 'xci-merge-{distro}-.*' + - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' block-level: 'NODE' @@ -102,7 +102,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'ANT' - project-pattern: '{project}' + project-pattern: 'releng-xci' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -110,6 +110,21 @@ forbidden-file-paths: - compare-type: ANT pattern: 'xci/scripts/vm/**' + - compare-type: ANT + pattern: 'docs/**' + - compare-type: ANT + pattern: 'prototypes/**' + - compare-type: ANT + pattern: 'upstream/**' + - project-compare-type: 'REG_EXP' + project-pattern: 'sfc|sdnvpn' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + file-paths: + - compare-type: ANT + pattern: 'scenarios/**' readable-message: true custom-url: '* $JOB_NAME $BUILD_URL' skip-vote: @@ -120,12 +135,15 @@ parameters: - project-parameter: - project: '{project}' + project: $GERRIT_PROJECT branch: '{branch}' - label: name: SLAVE_LABEL default: 'xci-virtual' - string: + name: DISTRO + default: '{distro}' + - string: name: CLEAN_DIB_IMAGES default: 'true' - string: @@ -134,8 +152,11 @@ description: 'Git URL to use on this Jenkins Slave' builders: + - 'xci-verify-set-scenario-macro' + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - multijob: name: deploy condition: SUCCESSFUL @@ -144,8 +165,8 @@ current-parameters: true predefined-parameters: | DISTRO={distro} - DEPLOY_SCENARIO=os-nosdn-nofeature CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -161,18 +182,27 @@ current-parameters: true predefined-parameters: | DISTRO={distro} - DEPLOY_SCENARIO=os-nosdn-nofeature CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + FUNCTEST_MODE=tier + FUNCTEST_TIER=healthcheck node-parameters: true kill-phase-on: NEVER abort-all-job: true + publishers: + - postbuildscript: + script-only-if-succeeded: false + script-only-if-failed: false + builders: + - shell: + !include-raw: ./xci-cleanup.sh + + - job-template: name: 'xci-verify-{distro}-{phase}-{type}-{stream}' @@ -194,14 +224,11 @@ parameters: - string: name: PROJECT - default: 'releng-xci' + default: $GERRIT_PROJECT - string: name: DISTRO default: 'ubuntu' - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature' - - string: name: FUNCTEST_MODE default: 'tier' - string: @@ -217,14 +244,13 @@ name: OPNFV_RELENG_DEV_PATH default: $WORKSPACE/ - string: - name: INSTALLER_TYPE - default: 'osa' - - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - ssh-agent-wrapper - build-timeout: timeout: 240 @@ -235,13 +261,19 @@ builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - 'xci-verify-{phase}-macro' # ------------------------------- # builder macros # ------------------------------- - builder: + name: 'xci-verify-set-scenario-macro' + builders: + - shell: + !include-raw: ./xci-set-scenario.sh + +- builder: name: 'xci-verify-deploy-macro' builders: - shell: @@ -254,4 +286,4 @@ name: 'xci-verify-healthcheck-macro' builders: - shell: - !include-raw: ./xci-cleanup.sh + !include-raw: ./xci-run-functest.sh diff --git a/jjb/yardstick/yardstick-daily-jobs.yml b/jjb/yardstick/yardstick-daily-jobs.yml index 481af2a42..20890fa13 100644 --- a/jjb/yardstick/yardstick-daily-jobs.yml +++ b/jjb/yardstick/yardstick-daily-jobs.yml @@ -136,7 +136,7 @@ <<: *euphrates # daisy CI PODs - baremetal: - slave-label: zte-pod3 + slave-label: daisy-baremetal installer: daisy auto-trigger-name: 'daily-trigger-disabled' <<: *master diff --git a/jjb/yardstick/yardstick-daily.sh b/jjb/yardstick/yardstick-daily.sh index f0e4f8e22..2fd68484e 100755 --- a/jjb/yardstick/yardstick-daily.sh +++ b/jjb/yardstick/yardstick-daily.sh @@ -50,7 +50,7 @@ opts="--privileged=true --rm" envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ -e NODE_NAME=${NODE_NAME} -e EXTERNAL_NETWORK=${EXTERNAL_NETWORK} \ -e YARDSTICK_BRANCH=${BRANCH} -e BRANCH=${BRANCH} \ - -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" + -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} -e CI_DEBUG=true" if [[ "${INSTALLER_TYPE}" == 'fuel' ]]; then envs+=" -e SSH_KEY=/root/.ssh/mcp.rsa" diff --git a/jjb/yardstick/yardstick-project-jobs.yml b/jjb/yardstick/yardstick-project-jobs.yml index e2fee2988..c03666388 100644 --- a/jjb/yardstick/yardstick-project-jobs.yml +++ b/jjb/yardstick/yardstick-project-jobs.yml @@ -132,7 +132,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running unit tests in Python 2.7 ..." cd $WORKSPACE @@ -146,7 +148,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running unit tests in Python 3 ..." cd $WORKSPACE @@ -160,7 +164,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running functional tests in Python 2.7 ..." cd $WORKSPACE @@ -174,7 +180,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running functional tests in Python 3 ..." cd $WORKSPACE @@ -188,7 +196,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running coverage tests ..." cd $WORKSPACE @@ -202,7 +212,9 @@ set -o errexit set -o pipefail + sudo apt-get -y autoremove sudo apt-get install -y build-essential python-dev python3-dev + sudo apt-get -y clean && sudo apt-get -y autoremove echo "Running style guidelines (PEP8) tests ..." cd $WORKSPACE diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh new file mode 100755 index 000000000..1984b4992 --- /dev/null +++ b/utils/artifacts.opnfv.org.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#export PATH=${PATH}:/root/gsutil + +#Step Generate index.html +if [ -f index.html ] ; then + rm -f index.html +fi + +OUTPUT="index.html" + +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs | grep -v review | awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +echo $index + echo "<LI><a href=\"${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT +done + +#functest logs########################## + +for project in functest vswitchperf +do + + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + index="$(echo ${index%/*} | sed s,/,_,g)" + echo "<LI><a href=\"http://artifacts.opnfv.org/${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT + done + +done +#End step 1 +##################################### + + +#genrate html files for all project except vswitchperf +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi +done + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +done + + + +#generate file for vswitch perf (I dont know what happend here but there is a wierd character in this bucket) + +index=vswitchperf +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + +done + + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +# Gerate html for logs + +for project in functest vswitchperf +do + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + + OUTPUT="$(echo ${index%/*}.html | sed s,/,_,g)" + echo $OUTPUT + rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + + + done + + + gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + + + done +done + + + +OUTPUT="index.html" +echo "<p> Generated on $(date) </p>" >> $OUTPUT + +cat <<EOF >> $OUTPUT +<script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); +ga('create', 'UA-831873-26', 'auto'); +ga('send', 'pageview'); +</script> +EOF + +#copy and uplad index file genrated in first step, last +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ diff --git a/utils/build-server-ansible/inventory.ini b/utils/build-server-ansible/inventory.ini new file mode 100644 index 000000000..115b1306e --- /dev/null +++ b/utils/build-server-ansible/inventory.ini @@ -0,0 +1,8 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +localhost ansible_connection=local diff --git a/utils/build-server-ansible/main.yml b/utils/build-server-ansible/main.yml new file mode 100644 index 000000000..0fcce715d --- /dev/null +++ b/utils/build-server-ansible/main.yml @@ -0,0 +1,37 @@ +############################################################################ +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +- hosts: "localhost" + become: "True" + tasks: + - debug: + msg: "{{ inventory_hostname }} is {{ ansible_distribution }}" + - include_vars: vars/defaults.yml + - include: vars/CentOS.yml + when: ansible_distribution == "CentOS" + - include: vars/Ubuntu.yml + when: ansible_distribution == "Ubuntu" + - name: Install Docker. + package: name={{ docker_package }} state={{ docker_package_state }} + - name: Ensure Docker is started and enabled at boot. + service: + name: docker + state: started + enabled: "yes" + - name: install gsutil + pip: + name: gsutil + state: present + - name: install tox + pip: + name: tox + state: present + - include: vars/docker-compose-CentOS.yml + when: ansible_distribution == "CentOS" + - include: vars/docker-compose-Ubuntu.yml + when: ansible_distribution == "Ubuntu" diff --git a/utils/build-server-ansible/vars/CentOS.yml b/utils/build-server-ansible/vars/CentOS.yml new file mode 100644 index 000000000..0d5a01111 --- /dev/null +++ b/utils/build-server-ansible/vars/CentOS.yml @@ -0,0 +1,72 @@ +############################################################################ +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################ +--- +- name: Ensure old versions of Docker are not installed. + package: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-common + - docker-engine + +- name: Add Docker GPG key. + rpm_key: + key: https://download.docker.com/linux/centos/gpg + state: present + +- name: Ensure epel is installed. + yum: + name: epel-release + state: present +- name: Ensure depdencies are installed. + yum: + name: "{{ item }}" + state: present + with_items: + - python-pip + - rpm-build + - kernel-headers + - libpcap-devel + - zlib-devel + - numactl-devel + - doxygen + - python-sphinx + - libvirt-devel + - python-devel + - openssl-devel + - python-six + - net-tools + - bc + +- name: install the 'Development tools' package group + yum: + name: "@Development tools" + state: present + +- name: Add Docker repository. + get_url: + url: "{{ docker_yum_repo_url }}" + dest: '/etc/yum.repos.d/docker-ce.repo' + owner: root + group: root + mode: 0644 + +- name: Configure Docker Edge repo. + ini_file: + dest: '/etc/yum.repos.d/docker-ce.repo' + section: 'docker-ce-edge' + option: enabled + value: '{{ docker_yum_repo_enable_edge }}' + +- name: Configure Docker Test repo. + ini_file: + dest: '/etc/yum.repos.d/docker-ce.repo' + section: 'docker-ce-test' + option: enabled + value: '{{ docker_yum_repo_enable_test }}' diff --git a/utils/build-server-ansible/vars/Ubuntu.yml b/utils/build-server-ansible/vars/Ubuntu.yml new file mode 100644 index 000000000..609c8d548 --- /dev/null +++ b/utils/build-server-ansible/vars/Ubuntu.yml @@ -0,0 +1,84 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +- name: Ensure old versions of Docker are not installed. + package: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-engine + +- name: Ensure depdencies are installed. + apt: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + - git + - build-essential + - curl + - wget + - rpm + - fuseiso + - createrepo + - genisoimage + - libfuse-dev + - dh-autoreconf + - pkg-config + - zlib1g-dev + - libglib2.0-dev + - libpixman-1-dev + - python-virtualenv + - python-dev + - libffi-dev + - libssl-dev + - libxml2-dev + - libxslt1-dev + - bc + - qemu-kvm + - libvirt-bin + - ubuntu-vm-builder + - bridge-utils + - monit + - openjdk-8-jre-headless + - python-nose + - dirmngr + - collectd + - flex + - bison + - libnuma-dev + - shellcheck + - python-pip + +- name: Add Docker apt key. + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + state: present + register: add_repository_key + ignore_errors: true + +- name: Ensure curl is present (on older systems without SNI). + package: name=curl state=present + when: add_repository_key|failed + +- name: Add Docker apt key (alternative for older systems without SNI). + # yamllint disable rule:line-length + shell: "curl -sSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -" + # yamllint enable rule:line-length + args: + warn: "no" + when: add_repository_key|failed + +- name: Add Docker repository. + apt_repository: + repo: "{{ docker_apt_repository }}" + state: present + update_cache: "yes" diff --git a/utils/build-server-ansible/vars/defaults.yml b/utils/build-server-ansible/vars/defaults.yml new file mode 100644 index 000000000..8d83380dd --- /dev/null +++ b/utils/build-server-ansible/vars/defaults.yml @@ -0,0 +1,23 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +docker_package: "docker-ce" +docker_package_state: present + +# Used only for Debian/Ubuntu. Switch 'stable' to 'edge' if needed. +docker_apt_release_channel: stable +# yamllint disable rule:line-length +docker_apt_repository: "deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} {{ docker_apt_release_channel }}" +# yamllint enable rule:line-length + +# Used only for RedHat/CentOS. +# yamllint disable rule:line-length +docker_yum_repo_url: https://download.docker.com/linux/centos/docker-ce.repo +# yamllint enable rule:line-length +docker_yum_repo_enable_edge: 0 +docker_yum_repo_enable_test: 0 diff --git a/utils/build-server-ansible/vars/docker-compose-CentOS.yml b/utils/build-server-ansible/vars/docker-compose-CentOS.yml new file mode 100644 index 000000000..fc4bcba7e --- /dev/null +++ b/utils/build-server-ansible/vars/docker-compose-CentOS.yml @@ -0,0 +1,12 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- name: Ensure docker compose is installed. + yum: + name: 'docker-compose' + state: present diff --git a/utils/build-server-ansible/vars/docker-compose-Ubuntu.yml b/utils/build-server-ansible/vars/docker-compose-Ubuntu.yml new file mode 100644 index 000000000..f985b6a4d --- /dev/null +++ b/utils/build-server-ansible/vars/docker-compose-Ubuntu.yml @@ -0,0 +1,12 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +- name: Ensure docker compose is installed + apt: + name: 'docker-compose' + state: present diff --git a/utils/fetch_k8_conf.sh b/utils/fetch_k8_conf.sh index f82fa5497..32c07f8b6 100755 --- a/utils/fetch_k8_conf.sh +++ b/utils/fetch_k8_conf.sh @@ -52,12 +52,14 @@ else touch $dest_path || error "Cannot create the file specified. Check that the path is correct and run the script again." fi -info "Fetching admin.conf file..." if [ "$installer_type" == "compass" ]; then + info "Fetching admin.conf file on Compass" sudo docker cp compass-tasks:/opt/admin.conf $dest_path &> /dev/null sudo chown $(whoami):$(whoami) $dest_path info "Fetch admin.conf successfully" +elif [ "$installer_type" == "joid" ]; then + info "Do nothing, config file has been provided in $HOME/joid_config/config for joid" else - error "Installer $installer is not supported by this script" + error "Installer $installer_type is not supported by this script" fi diff --git a/utils/jenkins-jnlp-connect.sh b/utils/jenkins-jnlp-connect.sh index cd81f29d3..0647d3210 100755 --- a/utils/jenkins-jnlp-connect.sh +++ b/utils/jenkins-jnlp-connect.sh @@ -11,10 +11,13 @@ #Monit setup script for opnfv jnlp slave connections test_firewall() { - echo "testing that the firewall is open for us at build.opnfv.org" - test=$(echo "blah"| nc -w 4 build.opnfv.org 57387 > /dev/null 2>&1; echo $?) +jenkins_hostname="${jenkins_hostname:-build.opnfv.org}" + + + echo "testing that the firewall is open for us at $jenkins_hostname" + test=$(echo "blah"| nc -w 4 $jenkins_hostname 57387 > /dev/null 2>&1; echo $?) if [[ $test == 0 ]]; then - echo "Firewall is open for us at build.opnfv.org" + echo "Firewall is open for us at $jenkins_hostname" exit 0 else cat << EOF @@ -84,8 +87,14 @@ main () { if [ -d /etc/monit/conf.d ]; then monitconfdir="/etc/monit/conf.d/" + monitconfig="/etc/monit/monitrc" + #add start delay + sed -i '/^#.* delay /s/^#//' "$monitconfig" elif [ -d /etc/monit.d ]; then monitconfdir="/etc/monit.d" + monitconfig="/etc/monitrc" + #add start delay + sed -i '/^#.* delay /s/^#//' "$monitconfig" else echo "Could not determine the location of the monit configuration file." echo "Make sure monit is installed." @@ -132,7 +141,7 @@ depends on jenkins_piddir\ fi if [[ $started_monit == "true" ]]; then - wget --timestamping https://build.opnfv.org/ci/jnlpJars/slave.jar && true + wget --timestamping https://"$jenkins_hostname"/jnlpJars/slave.jar && true chown $jenkinsuser:$jenkinsuser slave.jar if [[ -f /var/run/$jenkinsuser/jenkins_jnlp_pid ]]; then @@ -176,6 +185,7 @@ usage: $0 [OPTIONS] -u set jenkins user -n set slave name -s set secret key + -l set host, default is build.opnfv.org/ci -t test the connection string by connecting without monit -f test firewall @@ -190,13 +200,14 @@ if [[ -z "$@" ]]; then usage fi -while getopts "j:u:n:s:htf" OPTION +while getopts "j:u:n:s:l:htf" OPTION do case $OPTION in j ) jenkinshome="$OPTARG" ;; u ) jenkinsuser="$OPTARG" ;; n ) slave_name="$OPTARG" ;; s ) slave_secret="$OPTARG";; + l ) jenkins_hostname="$OPTARG" ;; h ) usage ;; t ) started_monit=true skip_monit=true @@ -206,6 +217,6 @@ do esac done -connectionstring="java -jar slave.jar -jnlpUrl https://build.opnfv.org/ci/computer/"$slave_name"/slave-agent.jnlp -secret "$slave_secret" -noCertificateCheck " - +jenkins_hostname="${jenkins_hostname:-build.opnfv.org/ci}" +connectionstring="java -jar slave.jar -jnlpUrl https://"$jenkins_hostname"/computer/"$slave_name"/slave-agent.jnlp -secret "$slave_secret" -noCertificateCheck " main "$@" diff --git a/utils/slave-monitor-0.1.sh b/utils/slave-monitor-0.1.sh index 161aaef21..5201f93d6 100644..100755 --- a/utils/slave-monitor-0.1.sh +++ b/utils/slave-monitor-0.1.sh @@ -8,9 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -#This will put a bunch of files in the pwd. you have been warned. #Counts how long slaves have been online or offline - +#exec 2>/dev/null #Yes I know about jq curlcommand() { @@ -25,74 +24,66 @@ curl -s "https://build.opnfv.org/ci/computer/api/json?tree=computer\[displayName | sed s,\",,g } -if [ -f podoutput-current ]; then - cp podoutput-current podoutput-lastiteration -fi - -curlcommand > podoutput-current +curlcommand > /tmp/podoutput-current -declare -A slavescurrent slaveslastiteration +declare -A slavescurrent while read -r name status ; do slavescurrent["$name"]="$status" -done < <(cat podoutput-current) - -while read -r name status ; do - slaveslastiteration["$name"]=$status -done < <(cat podoutput-lastiteration) - +done < <(cat /tmp/podoutput-current) + +#haste bin stopped allowing post :( +#files=(*online) +#for ((i=0; i<${#files[@]}; i+=9)); do +#./eplot -d -r [-1:74][-1:30] -m ${files[i]} ${files[i+1]} ${files[i+2]} ${files[i+3]} ${files[i+4]} ${files[i+5]} ${files[i+6]} ${files[i+7]} ${files[i+8]} ${files[i+9]} +#done | ./haste.bash +## main () { + for slavename in "${!slavescurrent[@]}"; do - #Slave is online. Mark it down. + + #Slave is online. Mark it down. if [ "${slavescurrent[$slavename]}" == "false" ]; then - if [ -f "$slavename"-offline ]; then - echo "removing offline status from $slavename slave was offline for $(cat "$slavename"-offline ) iterations" - rm "$slavename"-offline - fi - - if ! [ -f "$slavename"-online ]; then - echo "1" > "$slavename"-online - elif [ -f "$slavename"-online ]; then - #read and increment slavename - read -r -d $'\x04' var < "$slavename"-online - ((var++)) - echo -n "ONLINE $slavename " - echo "for $var iterations" - echo "$var" > "$slavename"-online - fi - fi + if ! [ -f /tmp/"$slavename"-online ]; then + echo "1" > /tmp/"$slavename"-online + echo "new online slave file created $slavename ${slavescurrent[$slavename]} up for 1 iterations" + fi - #went offline since last iteration. - if [ "${slavescurrent[$slavename]}" == "false" ] && [ "${slaveslastiteration[$slavename]}" == "true" ]; then - echo "JUST WENT OFFLINE $slavename " - if [ -f "$slavename"-online ]; then - echo "removing online status from $slavename. slave was online for $(cat "$slavename"-online ) iterations" - rm "$slavename"-online - fi + #read and increment slavename + var="$(cat /tmp/"$slavename"-online |tail -n 1)" + if [[ "$var" == "0" ]]; then + echo "slave $slavename ${slavescurrent[$slavename]} back up for $var iterations" + fi + ((var++)) + echo "$var" >> /tmp/"$slavename"-online + unset var + echo "$slavename up $(cat /tmp/$slavename-online | tail -n 10 | xargs)" fi - #slave is offline + #slave is offline remove all points if [ "${slavescurrent[$slavename]}" == "true" ]; then - if ! [ -f "$slavename"-offline ]; then - echo "1" > "$slavename"-offline - fi - - if [ -f "$slavename"-offline ]; then - #read and increment slavename - read -r -d $'\x04' var < "$slavename"-offline - ((var++)) - echo "$var" > "$slavename"-offline - if [ "$var" -gt "30" ]; then - echo "OFFLINE FOR $var ITERATIONS REMOVE $slavename " - else - echo "OFFLINE $slavename FOR $var ITERATIONS " - fi - fi + if ! [ -f /tmp/"$slavename"-online ]; then + echo "0" > /tmp/"$slavename"-online + echo "new offline slave file created $slavename ${slavescurrent[$slavename]} up for 0 iterations" + + fi + var="$(cat /tmp/"$slavename"-online |tail -n 1)" + + if [[ "$var" != "0" ]]; then + echo "slave $slavename ${slavescurrent[$slavename]} was up for $var iterations" + echo "slave $slavename ${slavescurrent[$slavename]} has gone offline, was $var iterations now reset to 0" + fi + + echo "0" >> /tmp/"$slavename"-online + echo "$slavename down $(cat /tmp/$slavename-online | tail -n 10 | xargs)" + unset var + fi + done } -main +main | sort | column -t |