diff options
Diffstat (limited to 'jjb')
52 files changed, 2309 insertions, 1726 deletions
diff --git a/jjb/apex/apex-project-jobs.yml b/jjb/apex/apex-project-jobs.yml index 58072865d..b12b1015c 100644 --- a/jjb/apex/apex-project-jobs.yml +++ b/jjb/apex/apex-project-jobs.yml @@ -46,6 +46,10 @@ branch: '{branch}' - apex-parameter: gs-pathname: '{gs-pathname}' + - string: + name: GERRIT_REFSPEC + default: 'refs/heads/{branch}' + description: "JJB configured GERRIT_REFSPEC parameter" scm: - git-scm-gerrit diff --git a/jjb/apex/apex-verify-jobs.yml b/jjb/apex/apex-verify-jobs.yml new file mode 100644 index 000000000..c57ac1d60 --- /dev/null +++ b/jjb/apex/apex-verify-jobs.yml @@ -0,0 +1,364 @@ +--- +- project: + name: 'apex-verify-jobs' + project: 'apex' + jobs: + - 'apex-verify-{stream}' + - 'apex-verify-gate-{stream}' + - 'apex-verify-unit-tests-{stream}' + stream: + - master: &master + branch: '{stream}' + gs-pathname: '' + verify-scenario: 'os-odl-nofeature-ha' + disabled: false + - danube: &danube + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + verify-scenario: 'os-odl_l3-nofeature-ha' + disabled: true + - euphrates: &euphrates + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + verify-scenario: 'os-odl-nofeature-ha' + disabled: false + +# Unit Test +- job-template: + name: 'apex-verify-unit-tests-{stream}' + + node: 'apex-build-{stream}' + + concurrent: true + + disabled: '{obj:disabled}' + + parameters: + - apex-parameter: + gs-pathname: '{gs-pathname}' + - project-parameter: + project: '{project}' + branch: '{branch}' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'true' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: 'apex' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'false' + file-paths: + - compare-type: ANT + pattern: 'apex/tests/**' + forbidden-file-paths: + - compare-type: ANT + pattern: '*' + - compare-type: ANT + pattern: 'apex/*' + - compare-type: ANT + pattern: 'build/**' + - compare-type: ANT + pattern: 'lib/**' + - compare-type: ANT + pattern: 'config/**' + - compare-type: ANT + pattern: 'apex/build/**' + - compare-type: ANT + pattern: 'apex/common/**' + - compare-type: ANT + pattern: 'apex/inventory/**' + - compare-type: ANT + pattern: 'apex/network/**' + - compare-type: ANT + pattern: 'apex/overcloud/**' + - compare-type: ANT + pattern: 'apex/settings/**' + - compare-type: ANT + pattern: 'apex/undercloud/**' + - compare-type: ANT + pattern: 'apex/virtual/**' + + properties: + - logrotate-default + - throttle: + max-per-node: 1 + max-total: 10 + option: 'project' + + builders: + - apex-unit-test + +# Verify +- job-template: + name: 'apex-verify-{stream}' + + node: 'apex-virtual-master' + + concurrent: true + + disabled: '{obj:disabled}' + + project-type: 'multijob' + + parameters: + - apex-parameter: + gs-pathname: '{gs-pathname}/dev' + - project-parameter: + project: '{project}' + branch: '{branch}' + - string: + name: ARTIFACT_VERSION + default: dev + description: "Used for overriding the ARTIFACT_VERSION" + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'true' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: 'apex' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + file-paths: + - compare-type: ANT + pattern: '*' + - compare-type: ANT + pattern: 'apex/*' + - compare-type: ANT + pattern: 'build/**' + - compare-type: ANT + pattern: 'lib/**' + - compare-type: ANT + pattern: 'config/**' + - compare-type: ANT + pattern: 'apex/build/**' + - compare-type: ANT + pattern: 'apex/common/**' + - compare-type: ANT + pattern: 'apex/inventory/**' + - compare-type: ANT + pattern: 'apex/network/**' + - compare-type: ANT + pattern: 'apex/overcloud/**' + - compare-type: ANT + pattern: 'apex/settings/**' + - compare-type: ANT + pattern: 'apex/undercloud/**' + - compare-type: ANT + pattern: 'apex/virtual/**' + forbidden-file-paths: + - compare-type: ANT + pattern: 'apex/tests/**' + - compare-type: ANT + pattern: 'docs/**' + + properties: + - logrotate-default + - throttle: + max-per-node: 3 + max-total: 10 + option: 'project' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'apex-verify-unit-tests-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + git-revision: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'apex-build-{stream}' + current-parameters: false + predefined-parameters: | + ARTIFACT_VERSION=$ARTIFACT_VERSION + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + git-revision: true + - multijob: + name: deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'apex-virtual-{stream}' + current-parameters: false + predefined-parameters: | + ARTIFACT_VERSION=$ARTIFACT_VERSION + DEPLOY_SCENARIO={verify-scenario} + OPNFV_CLEAN=yes + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + git-revision: true + +# Verify Scenario Gate +- job-template: + name: 'apex-verify-gate-{stream}' + + node: 'apex-build-{stream}' + + concurrent: true + + disabled: '{obj:disabled}' + + project-type: 'multijob' + + parameters: + - apex-parameter: + gs-pathname: '{gs-pathname}/dev' + - project-parameter: + project: '{project}' + branch: '{branch}' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." + - string: + name: ARTIFACT_VERSION + default: dev + description: "Used for overriding the ARTIFACT_VERSION" + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - comment-added-contains-event: + comment-contains-value: '^Patch Set [0-9]+: Code-Review\+2.*start-gate-scenario:.*' + projects: + - project-compare-type: 'ANT' + project-pattern: 'apex' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'ci/**' + - compare-type: ANT + pattern: 'build/**' + - compare-type: ANT + pattern: 'lib/**' + - compare-type: ANT + pattern: 'config/**' + - compare-type: ANT + pattern: 'apex/**' + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-verify.*' + - 'apex-virtual.*' + - throttle: + max-per-node: 1 + max-total: 10 + option: 'project' + + builders: + - multijob: + name: deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'apex-deploy-virtual-{stream}' + current-parameters: false + predefined-parameters: | + ARTIFACT_VERSION=$ARTIFACT_VERSION + DEPLOY_SCENARIO=gate + OPNFV_CLEAN=yes + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + GERRIT_EVENT_COMMENT_TEXT=$GERRIT_EVENT_COMMENT_TEXT + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + git-revision: true + # yamllint disable rule:line-length + - shell: | + echo DEPLOY_SCENARIO=$(echo $GERRIT_EVENT_COMMENT_TEXT | grep start-gate-scenario | grep -Eo 'os-.*') > detected_scenario + # yamllint enable rule:line-length + - inject: + properties-file: detected_scenario + - multijob: + name: functest-smoke + condition: SUCCESSFUL + projects: + - name: 'functest-apex-virtual-suite-{stream}' + current-parameters: false + predefined-parameters: | + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + FUNCTEST_MODE=tier + FUNCTEST_TIER=healthcheck + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false + +######################## +# builder macros +######################## +- builder: + name: apex-unit-test + builders: + - shell: + !include-raw: ./apex-unit-test.sh diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 4ec6efbd4..43e234ce1 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -3,9 +3,6 @@ name: 'apex' project: 'apex' jobs: - - 'apex-verify-{stream}' - - 'apex-verify-gate-{stream}' - - 'apex-verify-unit-tests-{stream}' - 'apex-runner-cperf-{stream}' - 'apex-virtual-{stream}' - 'apex-deploy-{platform}-{stream}' @@ -29,7 +26,7 @@ baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'master' - disable_daily: true + disable_daily: false - euphrates: &euphrates branch: 'stable/euphrates' gs-pathname: '/euphrates' @@ -161,10 +158,6 @@ <<: *euphrates - 'os-nosdn-fdio-ha': <<: *euphrates - - 'os-odl-fdio-noha': - <<: *euphrates - - 'os-odl-fdio-ha': - <<: *euphrates - 'os-nosdn-bar-ha': <<: *euphrates - 'os-nosdn-bar-noha': @@ -183,10 +176,6 @@ <<: *euphrates - 'os-odl-sfc-ha': <<: *euphrates - - 'os-odl-fdio_dvr-noha': - <<: *euphrates - - 'os-odl-fdio_dvr-ha': - <<: *euphrates - 'os-nosdn-calipso-noha': <<: *euphrates @@ -194,332 +183,6 @@ - 'baremetal' - 'virtual' -# Unit Test -- job-template: - name: 'apex-verify-unit-tests-{stream}' - - node: '{build-slave}' - - concurrent: true - - parameters: - - apex-parameter: - gs-pathname: '{gs-pathname}' - - project-parameter: - project: '{project}' - branch: '{branch}' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'apex' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'false' - file-paths: - - compare-type: ANT - pattern: 'apex/tests/**' - forbidden-file-paths: - - compare-type: ANT - pattern: '*' - - compare-type: ANT - pattern: 'apex/*' - - compare-type: ANT - pattern: 'build/**' - - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' - - compare-type: ANT - pattern: 'apex/build/**' - - compare-type: ANT - pattern: 'apex/common/**' - - compare-type: ANT - pattern: 'apex/inventory/**' - - compare-type: ANT - pattern: 'apex/network/**' - - compare-type: ANT - pattern: 'apex/overcloud/**' - - compare-type: ANT - pattern: 'apex/settings/**' - - compare-type: ANT - pattern: 'apex/undercloud/**' - - compare-type: ANT - pattern: 'apex/virtual/**' - - properties: - - logrotate-default - - throttle: - max-per-node: 1 - max-total: 10 - option: 'project' - - builders: - - shell: - !include-raw-escape: ./apex-unit-test.sh - -# Verify -- job-template: - name: 'apex-verify-{stream}' - - node: '{virtual-slave}' - - concurrent: true - - project-type: 'multijob' - - parameters: - - apex-parameter: - gs-pathname: '{gs-pathname}/dev' - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: ARTIFACT_VERSION - default: dev - description: "Used for overriding the ARTIFACT_VERSION" - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'apex' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'true' - file-paths: - - compare-type: ANT - pattern: '*' - - compare-type: ANT - pattern: 'apex/*' - - compare-type: ANT - pattern: 'build/**' - - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' - - compare-type: ANT - pattern: 'apex/build/**' - - compare-type: ANT - pattern: 'apex/common/**' - - compare-type: ANT - pattern: 'apex/inventory/**' - - compare-type: ANT - pattern: 'apex/network/**' - - compare-type: ANT - pattern: 'apex/overcloud/**' - - compare-type: ANT - pattern: 'apex/settings/**' - - compare-type: ANT - pattern: 'apex/undercloud/**' - - compare-type: ANT - pattern: 'apex/virtual/**' - forbidden-file-paths: - - compare-type: ANT - pattern: 'apex/tests/**' - - compare-type: ANT - pattern: 'docs/**' - - properties: - - logrotate-default - - throttle: - max-per-node: 3 - max-total: 10 - option: 'project' - - builders: - - description-setter: - description: "Built on $NODE_NAME" - - multijob: - name: basic - condition: SUCCESSFUL - projects: - - name: 'apex-verify-unit-tests-{stream}' - current-parameters: false - predefined-parameters: | - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - - multijob: - name: build - condition: SUCCESSFUL - projects: - - name: 'apex-build-{stream}' - current-parameters: false - predefined-parameters: | - ARTIFACT_VERSION=$ARTIFACT_VERSION - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - - multijob: - name: deploy-virtual - condition: SUCCESSFUL - projects: - - name: 'apex-virtual-{stream}' - current-parameters: false - predefined-parameters: | - ARTIFACT_VERSION=$ARTIFACT_VERSION - DEPLOY_SCENARIO={verify-scenario} - OPNFV_CLEAN=yes - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - -# Verify Scenario Gate -- job-template: - name: 'apex-verify-gate-{stream}' - - node: '{virtual-slave}' - - concurrent: true - - project-type: 'multijob' - - parameters: - - apex-parameter: - gs-pathname: '{gs-pathname}/dev' - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from parameters macro." - - string: - name: ARTIFACT_VERSION - default: dev - description: "Used for overriding the ARTIFACT_VERSION" - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - comment-added-contains-event: - comment-contains-value: '^Patch Set [0-9]+: Code-Review\+2.*start-gate-scenario:.*' - projects: - - project-compare-type: 'ANT' - project-pattern: 'apex' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: 'ci/**' - - compare-type: ANT - pattern: 'build/**' - - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' - - compare-type: ANT - pattern: 'apex/**' - - properties: - - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-verify.*' - - 'apex-virtual.*' - - throttle: - max-per-node: 1 - max-total: 10 - option: 'project' - - builders: - - multijob: - name: deploy-virtual - condition: SUCCESSFUL - projects: - - name: 'apex-deploy-virtual-{stream}' - current-parameters: false - predefined-parameters: | - ARTIFACT_VERSION=$ARTIFACT_VERSION - DEPLOY_SCENARIO=gate - OPNFV_CLEAN=yes - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - GERRIT_EVENT_COMMENT_TEXT=$GERRIT_EVENT_COMMENT_TEXT - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - # yamllint disable rule:line-length - - shell: | - echo DEPLOY_SCENARIO=$(echo $GERRIT_EVENT_COMMENT_TEXT | grep start-gate-scenario | grep -Eo 'os-.*') > detected_scenario - # yamllint enable rule:line-length - - inject: - properties-file: detected_scenario - - multijob: - name: functest-smoke - condition: SUCCESSFUL - projects: - - name: 'functest-apex-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - - job-template: name: 'apex-runner-cperf-{stream}' @@ -650,6 +313,8 @@ - job-template: name: 'apex-virtual-{stream}' + node: 'apex-virtual-master' + project-type: 'multijob' disabled: false @@ -1871,22 +1536,6 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-fdio-noha-baremetal-euphrates' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - - name: 'apex-os-odl-fdio-ha-baremetal-euphrates' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - name: 'apex-os-nosdn-bar-ha-baremetal-euphrates' node-parameters: false current-parameters: false @@ -1959,22 +1608,6 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-fdio_dvr-noha-baremetal-euphrates' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - - name: 'apex-os-odl-fdio_dvr-ha-baremetal-euphrates' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - name: 'apex-os-nosdn-calipso-noha-baremetal-euphrates' node-parameters: false current-parameters: false @@ -1984,12 +1617,6 @@ abort-all-job: true git-revision: false - builder: - name: 'apex-unit-test' - builders: - - shell: - !include-raw: ./apex-unit-test.sh - -- builder: name: 'apex-upload-artifact' builders: - shell: @@ -2013,12 +1640,12 @@ - trigger: name: 'apex-master' triggers: - - timed: '0 3 1 1 7' + - timed: '0 0 1-31/2 * *' - trigger: name: 'apex-euphrates' triggers: - - timed: '0 12 * * *' + - timed: '0 0 2-30/2 * *' - trigger: name: 'apex-danube' diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index 31415499c..6cd9de282 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -3,9 +3,6 @@ name: 'apex' project: 'apex' jobs: - - 'apex-verify-{stream}' - - 'apex-verify-gate-{stream}' - - 'apex-verify-unit-tests-{stream}' - 'apex-runner-cperf-{stream}' - 'apex-virtual-{stream}' - 'apex-deploy-{platform}-{stream}' @@ -29,7 +26,7 @@ baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'master' - disable_daily: true + disable_daily: false - euphrates: &euphrates branch: 'stable/euphrates' gs-pathname: '/euphrates' @@ -62,332 +59,6 @@ - 'baremetal' - 'virtual' -# Unit Test -- job-template: - name: 'apex-verify-unit-tests-{stream}' - - node: '{build-slave}' - - concurrent: true - - parameters: - - apex-parameter: - gs-pathname: '{gs-pathname}' - - project-parameter: - project: '{project}' - branch: '{branch}' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'apex' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'false' - file-paths: - - compare-type: ANT - pattern: 'apex/tests/**' - forbidden-file-paths: - - compare-type: ANT - pattern: '*' - - compare-type: ANT - pattern: 'apex/*' - - compare-type: ANT - pattern: 'build/**' - - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' - - compare-type: ANT - pattern: 'apex/build/**' - - compare-type: ANT - pattern: 'apex/common/**' - - compare-type: ANT - pattern: 'apex/inventory/**' - - compare-type: ANT - pattern: 'apex/network/**' - - compare-type: ANT - pattern: 'apex/overcloud/**' - - compare-type: ANT - pattern: 'apex/settings/**' - - compare-type: ANT - pattern: 'apex/undercloud/**' - - compare-type: ANT - pattern: 'apex/virtual/**' - - properties: - - logrotate-default - - throttle: - max-per-node: 1 - max-total: 10 - option: 'project' - - builders: - - shell: - !include-raw-escape: ./apex-unit-test.sh - -# Verify -- job-template: - name: 'apex-verify-{stream}' - - node: '{virtual-slave}' - - concurrent: true - - project-type: 'multijob' - - parameters: - - apex-parameter: - gs-pathname: '{gs-pathname}/dev' - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: ARTIFACT_VERSION - default: dev - description: "Used for overriding the ARTIFACT_VERSION" - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'apex' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'true' - file-paths: - - compare-type: ANT - pattern: '*' - - compare-type: ANT - pattern: 'apex/*' - - compare-type: ANT - pattern: 'build/**' - - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' - - compare-type: ANT - pattern: 'apex/build/**' - - compare-type: ANT - pattern: 'apex/common/**' - - compare-type: ANT - pattern: 'apex/inventory/**' - - compare-type: ANT - pattern: 'apex/network/**' - - compare-type: ANT - pattern: 'apex/overcloud/**' - - compare-type: ANT - pattern: 'apex/settings/**' - - compare-type: ANT - pattern: 'apex/undercloud/**' - - compare-type: ANT - pattern: 'apex/virtual/**' - forbidden-file-paths: - - compare-type: ANT - pattern: 'apex/tests/**' - - compare-type: ANT - pattern: 'docs/**' - - properties: - - logrotate-default - - throttle: - max-per-node: 3 - max-total: 10 - option: 'project' - - builders: - - description-setter: - description: "Built on $NODE_NAME" - - multijob: - name: basic - condition: SUCCESSFUL - projects: - - name: 'apex-verify-unit-tests-{stream}' - current-parameters: false - predefined-parameters: | - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - - multijob: - name: build - condition: SUCCESSFUL - projects: - - name: 'apex-build-{stream}' - current-parameters: false - predefined-parameters: | - ARTIFACT_VERSION=$ARTIFACT_VERSION - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - - multijob: - name: deploy-virtual - condition: SUCCESSFUL - projects: - - name: 'apex-virtual-{stream}' - current-parameters: false - predefined-parameters: | - ARTIFACT_VERSION=$ARTIFACT_VERSION - DEPLOY_SCENARIO={verify-scenario} - OPNFV_CLEAN=yes - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - -# Verify Scenario Gate -- job-template: - name: 'apex-verify-gate-{stream}' - - node: '{virtual-slave}' - - concurrent: true - - project-type: 'multijob' - - parameters: - - apex-parameter: - gs-pathname: '{gs-pathname}/dev' - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from parameters macro." - - string: - name: ARTIFACT_VERSION - default: dev - description: "Used for overriding the ARTIFACT_VERSION" - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - comment-added-contains-event: - comment-contains-value: '^Patch Set [0-9]+: Code-Review\+2.*start-gate-scenario:.*' - projects: - - project-compare-type: 'ANT' - project-pattern: 'apex' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: 'ci/**' - - compare-type: ANT - pattern: 'build/**' - - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' - - compare-type: ANT - pattern: 'apex/**' - - properties: - - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-verify.*' - - 'apex-virtual.*' - - throttle: - max-per-node: 1 - max-total: 10 - option: 'project' - - builders: - - multijob: - name: deploy-virtual - condition: SUCCESSFUL - projects: - - name: 'apex-deploy-virtual-{stream}' - current-parameters: false - predefined-parameters: | - ARTIFACT_VERSION=$ARTIFACT_VERSION - DEPLOY_SCENARIO=gate - OPNFV_CLEAN=yes - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - GERRIT_EVENT_COMMENT_TEXT=$GERRIT_EVENT_COMMENT_TEXT - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - # yamllint disable rule:line-length - - shell: | - echo DEPLOY_SCENARIO=$(echo $GERRIT_EVENT_COMMENT_TEXT | grep start-gate-scenario | grep -Eo 'os-.*') > detected_scenario - # yamllint enable rule:line-length - - inject: - properties-file: detected_scenario - - multijob: - name: functest-smoke - condition: SUCCESSFUL - projects: - - name: 'functest-apex-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - - job-template: name: 'apex-runner-cperf-{stream}' @@ -518,6 +189,8 @@ - job-template: name: 'apex-virtual-{stream}' + node: 'apex-virtual-master' + project-type: 'multijob' disabled: false @@ -1294,12 +967,6 @@ {% endfor -%} - builder: - name: 'apex-unit-test' - builders: - - shell: - !include-raw: ./apex-unit-test.sh - -- builder: name: 'apex-upload-artifact' builders: - shell: @@ -1323,12 +990,12 @@ - trigger: name: 'apex-master' triggers: - - timed: '0 3 1 1 7' + - timed: '0 0 1-31/2 * *' - trigger: name: 'apex-euphrates' triggers: - - timed: '0 12 * * *' + - timed: '0 0 2-30/2 * *' - trigger: name: 'apex-danube' diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden index 3b6b48502..98b698dd8 100644 --- a/jjb/apex/scenarios.yaml.hidden +++ b/jjb/apex/scenarios.yaml.hidden @@ -29,8 +29,6 @@ euphrates: - 'os-ovn-nofeature-noha' - 'os-nosdn-fdio-noha' - 'os-nosdn-fdio-ha' - - 'os-odl-fdio-noha' - - 'os-odl-fdio-ha' - 'os-nosdn-bar-ha' - 'os-nosdn-bar-noha' - 'os-nosdn-nofeature-ha-ipv6' @@ -40,8 +38,6 @@ euphrates: - 'os-nosdn-kvm_ovs_dpdk-ha' - 'os-odl-sfc-noha' - 'os-odl-sfc-ha' - - 'os-odl-fdio_dvr-noha' - - 'os-odl-fdio_dvr-ha' - 'os-nosdn-calipso-noha' danube: - 'os-nosdn-nofeature-noha' diff --git a/jjb/barometer/barometer-build.sh b/jjb/barometer/barometer-build.sh index e40841bc2..5f78aae7a 100644 --- a/jjb/barometer/barometer-build.sh +++ b/jjb/barometer/barometer-build.sh @@ -11,6 +11,7 @@ echo cd ci ./install_dependencies.sh ./build_rpm.sh +cp utility/rpms_list $WORKSPACE cd $WORKSPACE # save information regarding artifact into file diff --git a/jjb/barometer/barometer-upload-artifact.sh b/jjb/barometer/barometer-upload-artifact.sh index 0f639b5d9..f05dc2af8 100644 --- a/jjb/barometer/barometer-upload-artifact.sh +++ b/jjb/barometer/barometer-upload-artifact.sh @@ -2,7 +2,7 @@ set -o nounset set -o pipefail -RPM_LIST=$WORKSPACE/ci/utilities/rpms_list +RPM_LIST=$WORKSPACE/rpms_list RPM_WORKDIR=$WORKSPACE/rpmbuild RPM_DIR=$RPM_WORKDIR/RPMS/x86_64/ cd $WORKSPACE/ diff --git a/jjb/ci_gate_security/anteater-clone-all-repos.sh b/jjb/ci_gate_security/anteater-clone-all-repos.sh deleted file mode 100755 index 8a9e73d85..000000000 --- a/jjb/ci_gate_security/anteater-clone-all-repos.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# SPDX-license-identifier: Apache-2.0 -set -o errexit -set -o pipefail -set -o nounset -export PATH=$PATH:/usr/local/bin/ - - -#WORKSPACE="$(pwd)" - -cd $WORKSPACE -if [ ! -d "$WORKSPACE/allrepos" ]; then - mkdir $WORKSPACE/allrepos -fi - -cd $WORKSPACE/allrepos - -declare -a PROJECT_LIST -EXCLUDE_PROJECTS="All-Projects|All-Users|securedlab" - -PROJECT_LIST=($(ssh gerrit.opnfv.org -p 29418 gerrit ls-projects | egrep -v $EXCLUDE_PROJECTS)) -echo "PROJECT_LIST=(${PROJECT_LIST[*]})" > $WORKSPACE/opnfv-projects.sh - -for PROJECT in ${PROJECT_LIST[@]}; do - echo "> Cloning $PROJECT" - if [ ! -d "$PROJECT" ]; then - git clone "https://gerrit.opnfv.org/gerrit/$PROJECT.git" - else - pushd "$PROJECT" > /dev/null - git pull -f - popd > /dev/null - fi -done diff --git a/jjb/ci_gate_security/anteater-security-audit-weekly.sh b/jjb/ci_gate_security/anteater-security-audit-weekly.sh index 11909636a..6caa13117 100644 --- a/jjb/ci_gate_security/anteater-security-audit-weekly.sh +++ b/jjb/ci_gate_security/anteater-security-audit-weekly.sh @@ -1,37 +1,50 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +ANTEATER_SCAN_PATCHSET="${ANTEATER_SCAN_PATCHSET:-true}" + +cd $WORKSPACE +REPORTDIR='.reports' +mkdir -p $REPORTDIR +# Ensure any user can read the reports directory +chmod 777 $REPORTDIR + +ANTEATER_FILES="--patchset /home/opnfv/anteater/$PROJECT/patchset" + +if [[ "$ANTEATER_SCAN_PATCHSET" == "true" ]]; then + echo "Generating patchset file to list changed files" + git diff HEAD^1 --name-only | sed "s#^#/home/opnfv/anteater/$PROJECT/#" > $WORKSPACE/patchset + echo "Changed files are" + echo "--------------------------------------------------------" + cat $WORKSPACE/patchset + echo "--------------------------------------------------------" +else + echo "Checking full project $PROJECT" + ANTEATER_FILES="--path /home/opnfv/anteater/$PROJECT" +fi + +vols="-v $WORKSPACE:/home/opnfv/anteater/$PROJECT -v $WORKSPACE/$REPORTDIR:/home/opnfv/anteater/$REPORTDIR" +envs="-e PROJECT=$PROJECT" -echo "--------------------------------------------------------" -vols="-v $WORKSPACE/allrepos/:/home/opnfv/anteater/allrepos/" echo "Pulling releng-anteater docker image" echo "--------------------------------------------------------" docker pull opnfv/releng-anteater echo "--------------------------------------------------------" -cmd="docker run -id $vols opnfv/releng-anteater /bin/bash" -echo "Running docker command $cmd" -container_id=$($cmd) -echo "Container ID is $container_id" -source $WORKSPACE/opnfv-projects.sh -for project in "${PROJECT_LIST[@]}" - -do - cmd="/home/opnfv/venv/bin/anteater --project testproj --path /home/opnfv/anteater/allrepos/$project" - echo "Executing command inside container" - echo "$cmd" - echo "--------------------------------------------------------" - docker exec $container_id $cmd > $WORKSPACE/"$project".securityaudit.log 2>&1 -done +cmd="docker run -i $envs $vols --rm opnfv/releng-anteater \ +/home/opnfv/venv/bin/anteater --project $PROJECT $ANTEATER_FILES" +echo "Running docker container" +echo "$cmd" +$cmd > $WORKSPACE/securityaudit.log 2>&1 exit_code=$? echo "--------------------------------------------------------" -echo "Stopping docker container with ID $container_id" -docker stop $container_id - - -#gsutil cp $WORKSPACE/securityaudit.log \ -# gs://$GS_URL/$PROJECT-securityaudit-weekly.log 2>&1 -# -#gsutil -m setmeta \ -# -h "Content-Type:text/html" \ -# -h "Cache-Control:private, max-age=0, no-transform" \ -# gs://$GS_URL/$PROJECT-securityaudit-weekly.log > /dev/null 2>&1 +echo "Docker container exited with code: $exit_code" +echo "--------------------------------------------------------" +exit 0 diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml index e09339a4b..ef0da7f3a 100644 --- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml +++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml @@ -9,9 +9,76 @@ project: anteaterfw + repo: + - apex + - apex-os-net-config + - apex-puppet-tripleo + - apex-tripleo-heat-templates + - armband + - auto + - availability + - bamboo + - barometer + - bottlenecks + - calipso + - clover + - compass-containers + - compass4nfv + - conductor + - container4nfv + - copper + - cperf + - daisy + - doctor + - domino + - dovetail + - dpacc + - enfv + - fastpathmetrics + - fds + - fuel + - functest + - ipv6 + - joid + - kvmfornfv + - models + - moon + - multisite + - netready + - nfvbench + - octopus + - onosfw + - openretriever + - opera + - opnfvdocs + - orchestra + - ovn4nfv + - ovno + - ovsnfv + - parser + - pharos + - pharos-tools + - promise + - qtip + - releng + - releng-anteater + - releng-testresults + - releng-utils + - releng-xci + - samplevnf + - sdnvpn + - securityscanning + - sfc + - snaps + - stor4nfv + - storperf + - ves + - vswitchperf + - yardstick + jobs: - 'opnfv-security-audit-verify-{stream}' - - 'opnfv-security-audit-weekly-{stream}' + - 'opnfv-security-audit-{repo}-weekly-{stream}' stream: - master: @@ -23,18 +90,26 @@ # job templates ######################## - job-template: - name: 'opnfv-security-audit-weekly-{stream}' + name: 'opnfv-security-audit-{repo}-weekly-{stream}' disabled: '{obj:disabled}' parameters: - - label: - name: SLAVE_LABEL - default: 'ericsson-build3' - description: 'Slave label on Jenkins' + - ericsson-build3-defaults + - string: + name: ANTEATER_SCAN_PATCHSET + default: "false" + description: "Have anteater scan patchsets (true) or full project (false)" - project-parameter: - project: releng + project: '{repo}' branch: '{branch}' + - string: + name: GERRIT_REFSPEC + default: 'refs/heads/{stream}' + description: "Default Gerrit ref git HEAD should point to" + + scm: + - git-scm-gerrit triggers: - timed: '@weekly' @@ -42,6 +117,13 @@ builders: - anteater-security-audit-weekly + publishers: + # defined in jjb/global/releng-macros.yml + - 'email-{repo}-ptl': + subject: 'OPNFV Security Scan Result: {repo}' + - workspace-cleanup: + fail-build: false + - job-template: name: 'opnfv-security-audit-verify-{stream}' @@ -81,7 +163,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|fuel|functest|octopus|pharos|releng|sandbox|yardstick' + project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|fuel|functest|octopus|pharos|releng|sandbox|yardstick|infra|ipv6|kvmfornfv|lsoapi|models|moon|multisite|netready' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -117,12 +199,8 @@ - shell: !include-raw: ./anteater-report-to-gerrit.sh -# yamllint disable rule:indentation - builder: name: anteater-security-audit-weekly builders: - shell: - !include-raw: - - ./anteater-clone-all-repos.sh - - ./anteater-security-audit-weekly.sh -# yamllint enable rule:indentation + !include-raw: ./anteater-security-audit-weekly.sh diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 4adfc2a3f..446d0717a 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -13,7 +13,7 @@ gs-pathname: '' ppa-pathname: '/{stream}' disabled: false - openstack-version: ocata + openstack-version: pike euphrates: &euphrates stream: euphrates branch: 'stable/{stream}' diff --git a/jjb/container4nfv/container4nfv-project.yml b/jjb/container4nfv/container4nfv-project.yml index ffdbf9551..58070e1c5 100644 --- a/jjb/container4nfv/container4nfv-project.yml +++ b/jjb/container4nfv/container4nfv-project.yml @@ -31,6 +31,12 @@ - project-parameter: project: '{project}' branch: '{branch}' + # yamllint disable rule:line-length + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + # yamllint enable rule:line-length - 'opnfv-build-ubuntu-defaults' scm: @@ -78,10 +84,19 @@ properties: - logrotate-default + wrappers: + - fix-workspace-permissions + parameters: - project-parameter: project: '{project}' branch: '{branch}' + # yamllint disable rule:line-length + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + # yamllint enable rule:line-length - node: name: SLAVE_NAME description: 'Slave name on Jenkins' @@ -97,6 +112,6 @@ - timed: '@midnight' builders: - - shell: + - shell: | cd $WORKSPACE/ci ./deploy.sh diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index f2626db1c..090d2e1f2 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -93,7 +93,9 @@ - build-blocker: use-build-blocker: true blocking-jobs: + - 'daisy-os-.*?-{pod}-daily-.*?' - 'daisy-daily-.*' + - 'daisy-kolla-build-.*' block-level: 'NODE' wrappers: @@ -177,6 +179,7 @@ use-build-blocker: true blocking-jobs: - 'daisy.*-deploy-({pod})?-daily-.*' + - 'daisy-kolla-build-.*' block-level: 'NODE' parameters: diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index b6e9526e3..006153f6f 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -37,6 +37,72 @@ jobs: - '{installer}-daily-{stream}' - '{installer}-{phase}-daily-{stream}' + - '{installer}-kolla-build-{stream}' + +############################# +# docker build job templates +############################# +- job-template: + name: '{installer}-kolla-build-{stream}' + disabled: false + concurrent: true + + properties: + - logrotate-default + - throttle: + enabled: true + max-total: 4 + option: 'project' + - build-blocker: + use-build-blocker: true + # Note: Need to block all jobs which may create daisy VM. + blocking-jobs: + - '{installer}-kolla-build-.*' + - 'daisy-deploy-.*' + - 'daisy-daily-.*' + block-level: 'NODE' + + scm: + - git-scm + + triggers: + - 'daisy-kolla-build-{stream}-trigger' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'daisy-virtual-defaults' + - '{installer}-defaults' + - '{installer}-project-parameter': + gs-pathname: '{gs-pathname}' + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 720 + fail: true + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - shell: + !include-raw-escape: ./daisy4nfv-build-kolla-image.sh + + publishers: + - '{installer}-recipients' + - email-jenkins-admins-on-failure + +- trigger: + name: 'daisy-kolla-build-euphrates-trigger' + triggers: + - timed: '0 0 * * 0' + +- trigger: + name: 'daisy-kolla-build-master-trigger' + triggers: + - timed: '0 12 * * 0' + ######################## # job templates @@ -60,6 +126,7 @@ use-build-blocker: true blocking-jobs: - '{installer}-daily-.*' + - '{installer}-kolla-build-.*' - 'daisy4nfv-merge-build-.*' - 'daisy4nfv-verify-build-.*' block-level: 'NODE' diff --git a/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh b/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh new file mode 100755 index 000000000..9a1e2fc20 --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh @@ -0,0 +1,94 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +set -o errexit +set -o nounset +set -o pipefail + +importkey () { + # clone releng repository + echo "Cloning releng repository..." + [ -d releng ] && rm -rf releng + git clone https://gerrit.opnfv.org/gerrit/releng ./releng/ &> /dev/null + #this is where we import the siging key + if [ -f ./releng/utils/gpg_import_key.sh ]; then + source ./releng/utils/gpg_import_key.sh + fi +} + +upload_image_to_opnfv () { + image=$1 + + importkey + if gpg2 --list-keys | grep "opnfv-helpdesk@rt.linuxfoundation.org"; then + echo "Signing Key avaliable" + SIGN_ARTIFACT="true" + fi + + if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then + gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig $image + gsutil cp $image.sig gs://$GS_URL/upstream/$image.sig + echo "Image signature upload complete!" + fi + + sha512sum -b $image > $image.sha512sum + gsutil cp $image.sha512sum gs://$GS_URL/upstream/$image.sha512sum + + echo "Uploading $INSTALLER_TYPE artifact. This could take some time..." + echo + gsutil cp $image gs://$GS_URL/upstream/$image + gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/upstream/$image + + # check if we uploaded the file successfully to see if things are fine + gsutil ls gs://$GS_URL/upstream/$image + if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + exit 1 + fi +} + + + +echo "--------------------------------------------------------" +echo "This is diasy4nfv kolla image build job!" +echo "--------------------------------------------------------" + +# start the build +cd $WORKSPACE +rm -rf docker_build_dir +mkdir -p docker_build_dir + +# -j is for deciding which branch will be used when building, +# only for OPNFV +sudo -E ./ci/kolla-build.sh -j $JOB_NAME -w $WORKSPACE/docker_build_dir + +if [ $? -ne 0 ]; then + echo + echo "Kolla build failed!" + deploy_ret=1 +else + echo + echo "--------------------------------------------------------" + echo "Kolla build done!" +fi + +image=$(ls $WORKSPACE/docker_build_dir/kolla-build-output/kolla-image-*.tgz) +upload_image_to_opnfv $image + +echo +echo "--------------------------------------------------------" +echo "All done!" diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 6a04c5fbc..e0de9aa2e 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -25,6 +25,12 @@ # slave-label: 'ool-virtual3' # pod: 'ool-virtual3' + phase: + - 'build-x86_64': + slave-label: 'doctor-{installer}-verify' + - 'build-aarch64': + slave-label: 'doctor-{installer}-{phase}-verify' + inspector: - 'sample' - 'congress' @@ -37,22 +43,23 @@ auto-trigger-name: 'doctor-verify' is-python: true - pod: - - armband-baremetal: - slave-label: '{pod}' - - armband-virtual: - slave-label: '{pod}' + exclude: + - installer: 'apex' + phase: 'build-aarch64' jobs: - 'doctor-verify-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{pod}-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{stream}' + - 'doctor-verify-unit-test-{stream}' + - 'doctor-{task}-{inspector}-{stream}' + - 'doctor-{task}-{installer}-{inspector}-{phase}-{stream}' - job-template: name: 'doctor-verify-{stream}' disabled: '{obj:disabled}' + project-type: 'multijob' + parameters: - project-parameter: project: '{project}' @@ -86,50 +93,63 @@ pattern: 'doctor_tests/**' builders: - - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'doctor-build-and-unittest' + execution-type: PARALLEL + projects: + - name: 'doctor-verify-unit-test-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{pod}-{stream}' - - node: '{slave-label}' - + name: 'doctor-verify-unit-test-{stream}' disabled: '{obj:disabled}' - + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 60 parameters: - project-parameter: project: '{project}' branch: '{branch}' - 'opnfv-build-ubuntu-defaults' - scm: - git-scm-gerrit - - - triggers: - - '{auto-trigger-name}': - project: '{project}' - branch: '{branch}' - files: 'doctor_tests/**' - builders: - - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" + - 'doctor-verify-unit-test-builders-macro' + publishers: + - 'doctor-verify-unit-test-publishers-macro' - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{stream}' + name: 'doctor-{task}-{inspector}-{stream}' - node: '{slave-label}' + disabled: '{obj:disabled}' + + project-type: 'multijob' parameters: - project-parameter: project: '{project}' branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' - string: name: OS_CREDS default: /home/jenkins/openstack.creds description: 'OpenStack credentials' - - '{slave-label}-defaults' - - '{installer}-defaults' - string: name: DOCKER_TAG default: '{docker-tag}' @@ -151,13 +171,17 @@ default: 'doctor-notification' - string: name: TESTCASE_OPTIONS + # yamllint disable rule:line-length default: '-e INSPECTOR_TYPE={inspector} -e PYTHON_ENABLE={is-python} -v $WORKSPACE:/home/opnfv/repos/doctor' + # yamllint enable rule:line-length description: 'Addtional parameters specific to test case(s)' # functest-parameter - string: name: GS_PATHNAME default: '{gs-pathname}' + # yamllint disable rule:line-length description: "Version directory where the opnfv documents will be stored in gs repository" + # yamllint enable rule:line-length - string: name: FUNCTEST_REPO_DIR default: "/home/opnfv/repos/functest" @@ -181,7 +205,90 @@ files: 'doctor_tests/**' builders: + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'doctor-verify-apex-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-{task}-apex-{inspector}-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - multijob: + name: 'doctor-verify-fuel-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-{task}-fuel-{inspector}-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - multijob: + name: 'doctor-verify-fuel-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-{task}-fuel-{inspector}-build-aarch64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + +- job-template: + name: 'doctor-{task}-{installer}-{inspector}-{phase}-{stream}' + disabled: '{obj:disabled}' + node: '{slave-label}' + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 30 + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{installer}-defaults' + - '{slave-label}-defaults' + scm: + - git-scm-gerrit + builders: + - 'doctor-verify-installer-inspector-builders-macro' + publishers: + - 'doctor-verify-publishers-macro' +# ------------------------------- +# builder macros +# ------------------------------- + +- builder: + name: 'doctor-verify-unit-test-builders-macro' + builders: + - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" +- builder: + name: 'doctor-verify-installer-inspector-builders-macro' + builders: - 'clean-workspace-log' + # yamllint disable rule:line-length - shell: | # NOTE: Create symbolic link, so that we can archive file outside # of $WORKSPACE . @@ -200,14 +307,23 @@ # NOTE: checking the test result, as the previous job could return # 0 regardless the result of doctor test scenario. grep -e 'doctor test successfully' $functest_log || exit 1 + # yamllint enable rule:line-length +# ------------------------------- +# publisher macros +# ------------------------------- +- publisher: + name: 'doctor-verify-publishers-macro' publishers: - archive: artifacts: 'doctor_tests/*.log' - archive: artifacts: 'functest_results/$FUNCTEST_SUITE_NAME.log' - email-jenkins-admins-on-failure - +- publisher: + name: 'doctor-verify-unit-test-publishers-macro' + publishers: + - email-jenkins-admins-on-failure ##################################### # trigger macros diff --git a/jjb/dovetail/dovetail-ci-jobs.yml b/jjb/dovetail/dovetail-ci-jobs.yml index 6efe12820..99867c327 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yml +++ b/jjb/dovetail/dovetail-ci-jobs.yml @@ -25,7 +25,13 @@ branch: 'stable/{stream}' dovetail-branch: master gs-pathname: '/{stream}' - docker-tag: 'cvp.0.8.0' + docker-tag: 'cvp.0.9.0' + euphrates: &euphrates + stream: euphrates + branch: 'stable/{stream}' + dovetail-branch: master + gs-pathname: '/{stream}' + docker-tag: 'cvp.0.9.0' # ---------------------------------- # POD, PLATFORM, AND BRANCH MAPPING @@ -60,6 +66,16 @@ SUT: fuel auto-trigger-name: 'daily-trigger-disabled' <<: *danube + - baremetal: + slave-label: fuel-baremetal + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - virtual: + slave-label: fuel-virtual + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates # compass CI PODs - baremetal: slave-label: compass-baremetal @@ -81,6 +97,16 @@ SUT: compass auto-trigger-name: 'daily-trigger-disabled' <<: *danube + - baremetal: + slave-label: compass-baremetal + SUT: compass + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - virtual: + slave-label: compass-virtual + SUT: compass + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates # ------------------------------- # Installers not using labels # CI PODs @@ -108,6 +134,16 @@ SUT: apex auto-trigger-name: 'daily-trigger-disabled' <<: *danube + - virtual: + slave-label: apex-virtual-master + SUT: apex + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - baremetal: + slave-label: apex-baremetal-master + SUT: apex + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates # armband CI PODs - armband-baremetal: slave-label: armband-baremetal @@ -129,6 +165,16 @@ SUT: fuel auto-trigger-name: 'daily-trigger-disabled' <<: *danube + - armband-baremetal: + slave-label: armband-baremetal + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - armband-virtual: + slave-label: armband-virtual + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates # ------------------------------- # None-CI PODs # ------------------------------- @@ -172,10 +218,29 @@ SUT: compass auto-trigger-name: 'daily-trigger-disabled' <<: *danube + - zte-pod1: + slave-label: zte-pod1 + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - zte-pod3: + slave-label: zte-pod3 + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - huawei-pod4: + slave-label: huawei-pod4 + SUT: apex + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates + - huawei-pod7: + slave-label: huawei-pod7 + SUT: compass + auto-trigger-name: 'daily-trigger-disabled' + <<: *euphrates # ------------------------------- testsuite: - - 'compliance_set' - 'proposed_tests' jobs: diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 2cbb94766..e084e4bd0 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -24,6 +24,7 @@ mkdir -p ${DOVETAIL_CONFIG} sshkey="" # The path of openrc.sh is defined in fetch_os_creds.sh OPENRC=${DOVETAIL_CONFIG}/env_config.sh +CACERT=${DOVETAIL_CONFIG}/os_cacert if [[ ${INSTALLER_TYPE} == 'apex' ]]; then instack_mac=$(sudo virsh domiflist undercloud | grep default | \ grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") @@ -56,12 +57,22 @@ if [[ ${INSTALLER_TYPE} != 'joid' ]]; then echo "dovetail branch is $BRANCH" BRANCH_BACKUP=$BRANCH export BRANCH=$SUT_BRANCH - ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} + ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${CACERT} >${redirect} export BRANCH=$BRANCH_BACKUP fi if [[ -f $OPENRC ]]; then echo "INFO: openstack credentials path is $OPENRC" + if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then + if [[ -f ${CACERT} ]]; then + echo "INFO: ${INSTALLER_TYPE} openstack cacert file is ${CACERT}" + echo "export OS_CACERT=${CACERT}" >> ${OPENRC} + else + echo "ERROR: Can't find ${INSTALLER_TYPE} openstack cacert file. Please check if it is existing." + sudo ls -al ${DOVETAIL_CONFIG} + exit 1 + fi + fi cat $OPENRC else echo "ERROR: cannot find file $OPENRC. Please check if it is existing." @@ -69,37 +80,51 @@ else exit 1 fi -set +e - -sudo pip install virtualenv - -cd ${releng_repo}/modules -sudo virtualenv venv -source venv/bin/activate -sudo pip install -e ./ >/dev/null -sudo pip install netaddr +if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then + cat << EOF >${DOVETAIL_CONFIG}/pod.yaml +nodes: +- {ip: 10.1.0.52, name: node1, password: root, role: controller, user: root} +- {ip: 10.1.0.51, name: node2, password: root, role: controller, user: root} +- {ip: 10.1.0.50, name: node3, password: root, role: controller, user: root} +- {ip: 10.1.0.54, name: node4, password: root, role: compute, user: root} +- {ip: 10.1.0.53, name: node5, password: root, role: compute, user: root} -if [[ ${INSTALLER_TYPE} == compass ]]; then - options="-u root -p root" -elif [[ ${INSTALLER_TYPE} == fuel ]]; then - options="-u root -p r00tme" -elif [[ ${INSTALLER_TYPE} == apex ]]; then - options="-u stack -k /root/.ssh/id_rsa" -else - echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently." - echo "HA test cases may not run properly." +EOF fi -cmd="sudo python ${releng_repo}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \ - -i ${INSTALLER_IP} ${options} -f ${DOVETAIL_CONFIG}/pod.yaml" -echo ${cmd} -${cmd} +if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then + set +e + + sudo pip install virtualenv + + cd ${releng_repo}/modules + sudo virtualenv venv + source venv/bin/activate + sudo pip install -e ./ >/dev/null + sudo pip install netaddr + + if [[ ${INSTALLER_TYPE} == compass ]]; then + options="-u root -p root" + elif [[ ${INSTALLER_TYPE} == fuel ]]; then + options="-u root -p r00tme" + elif [[ ${INSTALLER_TYPE} == apex ]]; then + options="-u stack -k /root/.ssh/id_rsa" + else + echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently." + echo "HA test cases may not run properly." + fi -deactivate + cmd="sudo python ${releng_repo}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \ + -i ${INSTALLER_IP} ${options} -f ${DOVETAIL_CONFIG}/pod.yaml" + echo ${cmd} + ${cmd} -set -e + deactivate -cd ${WORKSPACE} + set -e + + cd ${WORKSPACE} +fi if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then echo "file ${DOVETAIL_CONFIG}/pod.yaml:" diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index bd2bdf4a8..735ca6f7f 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -4,10 +4,30 @@ set -e set +u set +o pipefail +check_os_deployment() { + FUNCTEST_IMAGE=opnfv/functest-healthcheck:${DOCKER_TAG} + cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} check_deployment" + eval ${cmd} + ret_value=$? + if [ ${ret_value} != 0 ]; then + echo "ERROR: Problem while checking OpenStack deployment." + exit 1 + else + echo "OpenStack deployment OK." + fi + +} + + run_tiers() { tiers=$1 - cmd_opt="prepare_env start && run_tests -r -t all" - [[ $BUILD_TAG =~ "suite" ]] && cmd_opt="prepare_env start && run_tests -t all" + if [[ ${BRANCH##*/} == "master" ]]; then + cmd_opt="run_tests -r -t all" + [[ $BUILD_TAG =~ "suite" ]] && cmd_opt="run_tests -t all" + else + cmd_opt="prepare_env start && run_tests -r -t all" + [[ $BUILD_TAG =~ "suite" ]] && cmd_opt="prepare_env start && run_tests -t all" + fi ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value" echo 0 > ${ret_val_file} @@ -15,7 +35,7 @@ run_tiers() { FUNCTEST_IMAGE=opnfv/functest-${tier}:${DOCKER_TAG} echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null - cmd="docker run --privileged=true ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" + cmd="docker run --rm --privileged=true ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest tier '${tier}'. CMD: ${cmd}" eval ${cmd} ret_value=$? @@ -31,21 +51,24 @@ run_tiers() { run_test() { test_name=$1 - cmd_opt="prepare_env start && run_tests -t ${test_name}" - [[ $BUILD_TAG =~ "suite" ]] && cmd_opt="prepare_env start && run_tests -t ${test_name}" + if [[ ${BRANCH##*/} == "master" ]]; then + cmd_opt="run_tests -t ${test_name}" + else + cmd_opt="prepare_env start && run_tests -t ${test_name}" + fi ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value" echo 0 > ${ret_val_file} # Determine which Functest image should be used for the test case case ${test_name} in connection_check|api_check|snaps_health_check) FUNCTEST_IMAGE=opnfv/functest-healthcheck:${DOCKER_TAG} ;; - vping_ssh|vping_userdata|tempest_smoke_serial|rally_sanity|refstack_defcore|odl|odl_netvirt|fds|snaps_smoke) + vping_ssh|vping_userdata|tempest_smoke_serial|rally_sanity|refstack_defcore|odl|odl_netvirt|snaps_smoke) FUNCTEST_IMAGE=opnfv/functest-smoke:${DOCKER_TAG} ;; - tempest_full_parallel|tempest_custom|rally_full) + tempest_full_parallel|rally_full) FUNCTEST_IMAGE=opnfv/functest-components:${DOCKER_TAG} ;; cloudify_ims|orchestra_openims|orchestra_clearwaterims|vyos_vrouter) FUNCTEST_IMAGE=opnfv/functest-vnf:${DOCKER_TAG} ;; - promise|doctor-notification|bgpvpn|functest-odl-sfc|domino-multinode|barometercollectd) + promise|doctor-notification|bgpvpn|functest-odl-sfc|domino-multinode|barometercollectd|fds) FUNCTEST_IMAGE=opnfv/functest-features:${DOCKER_TAG} ;; parser-basics) FUNCTEST_IMAGE=opnfv/functest-parser:${DOCKER_TAG} ;; @@ -56,7 +79,7 @@ run_test() { esac echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null - cmd="docker run --privileged=true ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" + cmd="docker run --rm --privileged=true ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest test case '${test_name}'. CMD: ${cmd}" eval ${cmd} ret_value=$? @@ -71,7 +94,7 @@ FUNCTEST_DIR=/home/opnfv/functest DEPLOY_TYPE=baremetal [[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt HOST_ARCH=$(uname -m) -DOCKER_TAG=${DOCKER_TAG:-latest} +DOCKER_TAG=`[[ ${BRANCH##*/} == "master" ]] && echo "latest" || echo ${BRANCH##*/}` # Prepare OpenStack credentials volume rc_file=${HOME}/opnfv-openrc.sh @@ -132,6 +155,7 @@ set +e if [[ ${DEPLOY_SCENARIO} =~ ^os-.* ]]; then + [[ ${BRANCH##*/} == "master" ]] && check_os_deployment if [ ${FUNCTEST_MODE} == 'testcase' ]; then echo "FUNCTEST_MODE=testcase, FUNCTEST_SUITE_NAME=${FUNCTEST_SUITE_NAME}" run_test ${FUNCTEST_SUITE_NAME} diff --git a/jjb/functest/functest-cleanup.sh b/jjb/functest/functest-cleanup.sh index fc277b9ed..c21b543a6 100755 --- a/jjb/functest/functest-cleanup.sh +++ b/jjb/functest/functest-cleanup.sh @@ -3,11 +3,7 @@ [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" echo "Cleaning up docker containers/images..." -HOST_ARCH=$(uname -m) FUNCTEST_IMAGE=opnfv/functest -if [ "$HOST_ARCH" = "aarch64" ]; then - FUNCTEST_IMAGE="${FUNCTEST_IMAGE}_${HOST_ARCH}" -fi # Remove containers along with image opnfv/functest*:<none> dangling_images=($(docker images -f "dangling=true" | grep $FUNCTEST_IMAGE | awk '{print $3}')) diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index 2fc70b916..2d5d397ae 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -14,12 +14,10 @@ stream: master branch: '{stream}' gs-pathname: '' - docker-tag: 'latest' euphrates: &euphrates stream: euphrates branch: 'stable/{stream}' gs-pathname: '/{stream}' - docker-tag: 'stable' # ------------------------------- # POD, INSTALLER, AND BRANCH MAPPING # ------------------------------- @@ -234,10 +232,6 @@ name: DEPLOY_SCENARIO default: 'os-odl_l2-nofeature-ha' - string: - name: DOCKER_TAG - default: '{docker-tag}' - description: 'Tag to pull docker image' - - string: name: CLEAN_DOCKER_IMAGES default: 'false' description: 'Remove downloaded docker images (opnfv/functest*:*)' @@ -399,8 +393,7 @@ !include-raw: - ./functest-env-presetup.sh - ../../utils/fetch_os_creds.sh - - ./set-functest-env.sh - - ./functest-loop.sh + - ./functest-alpine.sh # yamllint enable rule:indentation - builder: diff --git a/jjb/functest/functest-loop.sh b/jjb/functest/functest-loop.sh deleted file mode 100755 index a590d9f95..000000000 --- a/jjb/functest/functest-loop.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set +e - -[[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+="-r" -cmd="run_tests -t all ${flags}" - -container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) -docker exec $container_id $cmd - -ret_value=$? -ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value" -echo ${ret_value}>${ret_val_file} - -exit 0 diff --git a/jjb/functest/functest-weekly-jobs.yml b/jjb/functest/functest-weekly-jobs.yml index 6826e4151..5d572aacc 100644 --- a/jjb/functest/functest-weekly-jobs.yml +++ b/jjb/functest/functest-weekly-jobs.yml @@ -118,13 +118,11 @@ # yamllint disable rule:indentation - shell: !include-raw: - - ./functest-env-presetup.sh - - ../../utils/fetch_os_creds.sh - - ./set-functest-env.sh + - ./functest-env-presetup.sh + - ../../utils/fetch_os_creds.sh + - ./functest-alpine.sh # yamllint enable rule:indentation - shell: - !include-raw: ./functest-loop.sh - - shell: !include-raw: ../../utils/push-test-logs.sh - shell: !include-raw: ./functest-exit.sh diff --git a/jjb/functest/set-functest-env.sh b/jjb/functest/set-functest-env.sh deleted file mode 100755 index b116a8528..000000000 --- a/jjb/functest/set-functest-env.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -set -e -set +u -set +o pipefail - -[[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" - -DEPLOY_TYPE=baremetal -[[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt -HOST_ARCH=$(uname -m) - -# Prepare OpenStack credentials volume -rc_file_vol="-v ${HOME}/opnfv-openrc.sh:/home/opnfv/functest/conf/openstack.creds" - -if [[ ${INSTALLER_TYPE} == 'joid' ]]; then - rc_file_vol="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" -elif [[ ${INSTALLER_TYPE} == 'compass' && ${BRANCH} == 'master' ]]; then - cacert_file_vol="-v ${HOME}/os_cacert:/home/opnfv/functest/conf/os_cacert" - echo "export OS_CACERT=/home/opnfv/functest/conf/os_cacert" >> ${HOME}/opnfv-openrc.sh -elif [[ ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then - cacert_file_vol="-v ${HOME}/os_cacert:/etc/ssl/certs/mcp_os_cacert" -fi - - -# Set iptables rule to allow forwarding return traffic for container -if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then - sudo iptables -I FORWARD -j RETURN -fi - -echo "Functest: Start Docker and prepare environment" - -if [ "$BRANCH" != 'stable/danube' ]; then - echo "Functest: Download images that will be used by test cases" - images_dir="${HOME}/opnfv/functest/images" - chmod +x ${WORKSPACE}/functest/ci/download_images.sh - ${WORKSPACE}/functest/ci/download_images.sh ${images_dir} > ${redirect} - images_vol="-v ${images_dir}:/home/opnfv/functest/images" - echo "Functest: Images successfully downloaded" -fi - -dir_result="${HOME}/opnfv/functest/results/${BRANCH##*/}" -mkdir -p ${dir_result} -sudo rm -rf ${dir_result}/* -results_vol="-v ${dir_result}:/home/opnfv/functest/results" -custom_params= -test -f ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG} && custom_params=$(cat ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG}) -echo "Functest: custom parameters successfully retrieved: ${custom_params}" - -envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ - -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ - -e BUILD_TAG=${BUILD_TAG} -e CI_DEBUG=${CI_DEBUG} -e DEPLOY_TYPE=${DEPLOY_TYPE}" - -ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -if [ "${INSTALLER_TYPE}" == 'fuel' ] && [ "$BRANCH" != 'stable/danube' ]; then - COMPUTE_ARCH=$(ssh -l ubuntu ${INSTALLER_IP} -i ${SSH_KEY} ${ssh_options} \ - "sudo salt 'cmp*' grains.get cpuarch --out yaml | awk '{print \$2; exit}'") - envs="${envs} -e POD_ARCH=${COMPUTE_ARCH}" -fi - -if [[ ${INSTALLER_TYPE} == 'compass' && ${DEPLOY_SCENARIO} == *'os-nosdn-openo-ha'* ]]; then - openo_msb_port=${openo_msb_port:-80} - openo_msb_endpoint="$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ip} \ - 'mysql -ucompass -pcompass -Dcompass -e "select package_config from cluster;" \ - | sed s/,/\\n/g | grep openo_ip | cut -d \" -f 4'):$openo_msb_port" - - envs=${env}" -e OPENO_MSB_ENDPOINT=${openo_msb_endpoint}" -fi - -if [ "$BRANCH" != 'stable/danube' ]; then - volumes="${images_vol} ${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol} ${cacert_file_vol}" -else - volumes="${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" -fi - -echo "Functest: volumes defined" - -FUNCTEST_IMAGE="opnfv/functest" -if [ "$HOST_ARCH" = "aarch64" ]; then - FUNCTEST_IMAGE="${FUNCTEST_IMAGE}_${HOST_ARCH}" -fi - -echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE}:${DOCKER_TAG}" -docker pull ${FUNCTEST_IMAGE}:$DOCKER_TAG >/dev/null - -cmd="sudo docker run --privileged=true -id ${envs} ${volumes} \ - ${custom_params} ${TESTCASE_OPTIONS} \ - ${FUNCTEST_IMAGE}:${DOCKER_TAG} /bin/bash" -echo "Functest: Running docker run command: ${cmd}" -${cmd} >${redirect} -sleep 5 -container_id=$(docker ps | grep "${FUNCTEST_IMAGE}:${DOCKER_TAG}" | awk '{print $1}' | head -1) -echo "Container ID=${container_id}" -if [ -z ${container_id} ]; then - echo "Cannot find opnfv/functest container ID ${container_id}. Please check if it is existing." - docker ps -a - exit 1 -fi -echo "Starting the container: docker start ${container_id}" -docker start ${container_id} -sleep 5 -docker ps >${redirect} -if [ $(docker ps | grep "${FUNCTEST_IMAGE}:${DOCKER_TAG}" | wc -l) == 0 ]; then - echo "The container ${FUNCTEST_IMAGE} with ID=${container_id} has not been properly started. Exiting..." - exit 1 -fi - -cmd="prepare_env start" -echo "Executing command inside the docker: ${cmd}" -docker exec ${container_id} ${cmd} diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index e00047ba0..86f4c4265 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -196,17 +196,6 @@ sudo chown -R $USER:$USER $WORKSPACE || exit 1 - builder: - name: build-html-and-pdf-docs-output - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - git clone ssh://gerrit.opnfv.org:29418/opnfvdocs docs_build/_opnfvdocs - GERRIT_COMMENT=gerrit_comment.txt ./docs_build/_opnfvdocs/scripts/docs-build.sh - -- builder: name: upload-under-review-docs-to-opnfv-artifacts builders: - shell: | @@ -217,7 +206,7 @@ export PATH=$PATH:/usr/local/bin/ [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] - [[ -d docs_output ]] || exit 0 + [[ -d docs/_build/ ]] || exit 0 echo echo "###########################" @@ -230,7 +219,7 @@ local_path="upload/$GERRIT_CHANGE_NUMBER" mkdir -p upload - mv docs_output "$local_path" + mv docs/_build/html/ "$local_path" gsutil -m cp -r "$local_path" "gs://$gs_base" gsutil -m setmeta \ @@ -242,46 +231,6 @@ find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \ sed -e "s|^$local_path| http://$gs_path|" >> gerrit_comment.txt -- builder: - name: upload-generated-docs-to-opnfv-artifacts - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - [[ -d docs_output ]] || exit 0 - - echo - echo "########################" - echo "UPLOADING GENERATED DOCS" - echo "########################" - echo - - echo "gs_path="$GS_URL/docs"" - echo "local_path="upload/docs"" - - gs_path="$GS_URL/docs" - local_path="upload/docs" - - mkdir -p upload - mv docs_output "$local_path" - ls "$local_path" - - echo "gsutil -m cp -r "$local_path"/* "gs://$gs_path"" - gsutil -m cp -r "$local_path"/* "gs://$gs_path" - - gsutil -m setmeta \ - -h "Content-Type:text/html" \ - -h "Cache-Control:private, max-age=0, no-transform" \ - "gs://$gs_path"/**.html > /dev/null 2>&1 - - echo "Document link(s):" >> gerrit_comment.txt - find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \ - sed -e "s|^$local_path| http://$gs_path|" >> gerrit_comment.txt - # To take advantage of this macro, have your build write # out the file 'gerrit_comment.txt' with information to post # back to gerrit and include this macro in the list of builders. @@ -335,28 +284,6 @@ fi - builder: - name: build-and-upload-artifacts-json-api - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - export PATH=$PATH:/usr/local/bin/ - - virtualenv -p python2.7 $WORKSPACE/releng_artifacts - source $WORKSPACE/releng_artifacts/bin/activate - - # install python packages - pip install google-api-python-client - - # generate and upload index file - echo "Generating Artifacts API ..." - python $WORKSPACE/utils/opnfv-artifacts.py > index.json - gsutil cp index.json gs://artifacts.opnfv.org/index.json - - deactivate - -- builder: name: lint-python-code builders: - shell: | @@ -431,19 +358,10 @@ - builder: name: upload-review-docs builders: - - build-html-and-pdf-docs-output - upload-under-review-docs-to-opnfv-artifacts - report-build-result-to-gerrit - builder: - name: upload-merged-docs - builders: - - build-html-and-pdf-docs-output - - upload-generated-docs-to-opnfv-artifacts - - report-build-result-to-gerrit - - remove-old-docs-from-opnfv-artifacts - -- builder: name: check-bash-syntax builders: - shell: "find . -name '*.sh' | xargs bash -n" @@ -459,23 +377,29 @@ export PATH=$PATH:/usr/local/bin/ # install python packages - pip install "yamllint==1.6.0" + sudo pip install "yamllint==1.8.2" # generate and upload lint log echo "Running yaml code on $PROJECT ..." + # Get list of yaml files + YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true + + #If YAML_FILES is none exit with 0 + if [ -z "$YAML_FILES" ]; then + exit 0 + fi + # Ensure we start with a clean environment rm -f yaml-violation.log lint.log - # Get number of yaml violations. If none, this will be an - # empty string: "" - find . \ - -type f -name "*.yml" -print \ - -o -name "*.yaml" -print | \ - xargs yamllint > yaml-violation.log || true + # Yamllint files only in patchset + for yamlfile in $YAML_FILES; do + yamllint $yamlfile >> yaml-violation.log || true + done if [ -s "yaml-violation.log" ]; then - SHOWN=$(cat yaml-violation.log| grep -v "^$" |wc -l) + SHOWN=$(grep -c -v "^$" yaml-violation.log) echo -e "First $SHOWN shown\n---" > lint.log cat yaml-violation.log >> lint.log sed -r -i '4,$s/^/ /g' lint.log @@ -539,3 +463,501 @@ failure: true send-to: - recipients + +# Email PTL publishers +- email_ptl_defaults: &email_ptl_defaults + name: 'email_ptl_defaults' + content-type: text + attach-build-log: true + attachments: '*.log' + compress-log: true + always: true + subject: '{subject}' + +- publisher: &email_apex_ptl_defaults + name: 'email-apex-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + trozet@redhat.com +- publisher: + name: 'email-apex-os-net-config-ptl' + <<: *email_apex_ptl_defaults +- publisher: + name: 'email-apex-puppet-tripleo-ptl' + <<: *email_apex_ptl_defaults +- publisher: + name: 'email-apex-tripleo-heat-templates-ptl' + <<: *email_apex_ptl_defaults + +- publisher: + name: 'email-armband-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bob.monkman@arm.com + +- publisher: + name: 'email-auto-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + tina.tsou@arm.com + +- publisher: + name: 'email-availability-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fuqiao@chinamobile.com + +- publisher: + name: 'email-bamboo-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + donaldh@cisco.com + +- publisher: + name: 'email-barometer-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + aasmith@redhat.com + +- publisher: + name: 'email-bottlenecks-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + gabriel.yuyang@huawei.com + +- publisher: + name: 'email-calipso-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + korlev@cisco.com + +- publisher: + name: 'email-clover-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + stephen.wong1@huawei.com + +- publisher: &email_compass4nfv_ptl_defaults + name: 'email-compass4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + chigang@huawei.com +- publisher: + name: 'email-compass-containers-ptl' + <<: *email_compass4nfv_ptl_defaults + +- publisher: + name: 'email-conductor-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + limingjiang@huawei.com + +- publisher: + name: 'email-container4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + jiaxuan@chinamobile.com + +- publisher: + name: 'email-copper-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + aimeeu.opensource@gmail.com + +- publisher: + name: 'email-cperf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + matt.welch@intel.com + +- publisher: + name: 'email-daisy-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + hu.zhijiang@zte.com.cn + +- publisher: + name: 'email-doctor-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + r-mibu@cq.jp.nec.com + +- publisher: + name: 'email-domino-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ulas.kozat@huawei.com + +- publisher: + name: 'email-dovetail-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + hongbo.tianhongbo@huawei.com + +- publisher: + name: 'email-dpacc-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + denglingli@chinamobile.com + +- publisher: + name: 'email-enfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + JBuchanan@advaoptical.com + +- publisher: + name: 'email-escalator-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + kong.wei2@zte.com.cn + +- publisher: + name: 'email-fastpathmetrics-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + maryam.tahhan@intel.com + +- publisher: + name: 'email-fds-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fbrockne@cisco.com + +- publisher: + name: 'email-fuel-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + gelkinbard@mirantis.com + +- publisher: + name: 'email-functest-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + cedric.ollivier@orange.com + +- publisher: + name: 'email-ipv6-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bh526r@att.com + +- publisher: + name: 'email-joid-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + artur.tyloch@canonical.com + +- publisher: + name: 'email-kvmfornfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + raghuveer.reddy@intel.com + +- publisher: + name: 'email-models-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bs3131@att.com + +- publisher: + name: 'email-moon-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ruan.he@orange.com + +- publisher: + name: 'email-multisite-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + joehuang@huawei.com + +- publisher: + name: 'email-netready-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + georg.kunz@ericsson.com + +- publisher: + name: 'email-nfvbench-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ahothan@cisco.com + +- publisher: + name: 'email-octopus-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ulrich.kleber@huawei.com + +- publisher: + name: 'email-onosfw-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + su.wei@huawei.com + +- publisher: + name: 'email-openretriever-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + jiaxuan@chinamobile.com + +- publisher: + name: 'email-opera-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + Yingjun.li@huawei.com + +- publisher: + name: 'email-opnfvdocs-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + sofia.wallin@ericsson.com + +- publisher: + name: 'email-orchestra-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + giuseppe.carella@fokus.fraunhofer.de + +- publisher: + name: 'email-ovn4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + trinath.somanchi@gmail.com + +- publisher: + name: 'email-ovno-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + wsmackie@juniper.net + +- publisher: + name: 'email-ovsnfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + MarkD.Graymark.d.gray@intel.com + +- publisher: + name: 'email-parser-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + shang.xiaodong@zte.com.cn + +- publisher: &email_pharos_ptl_defaults + name: 'email-pharos-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + jack.morgan@intel.com +- publisher: + name: 'email-pharos-tools-ptl' + <<: *email_pharos_ptl_defaults + +- publisher: + name: 'email-promise-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + kunzmann@docomolab-euro.com + +- publisher: + name: 'email-qtip-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + wu.zhihui1@zte.com.cn + +- publisher: &email_releng_ptl_defaults + name: 'email-releng-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fatih.degirmenci@ericsson.com +- publisher: + name: 'email-releng-anteater-ptl' + <<: *email_releng_ptl_defaults +- publisher: + name: 'email-releng-testresults-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fatih.degirmenci@ericsson.com + feng.xiaowei@zte.com.cn +- publisher: + name: 'email-releng-utils-ptl' + <<: *email_releng_ptl_defaults +- publisher: + name: 'email-releng-xci-ptl' + <<: *email_releng_ptl_defaults + +- publisher: + name: 'email-samplevnf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + deepak.s@intel.com + +- publisher: + name: 'email-sdnvpn-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + tim.irnich@ericsson.com + +- publisher: + name: 'email-securityscanning-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + lhinds@redhat.com + +- publisher: + name: 'email-sfc-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ManuelBuilmbuil@suse.com + +- publisher: + name: 'email-snaps-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + s.pisarski@cablelabs.com + +- publisher: + name: 'email-stor4nfv-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + shane.wang@intel.com + +- publisher: + name: 'email-storperf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + mark.beierl@emc.com + +- publisher: + name: 'email-ves-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + bryan.sullivan@att.com + +- publisher: + name: 'email-vswitchperf-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + sridhar.rao@spirent.com + +- publisher: + name: 'email-yardstick-ptl' + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + ross.b.brattain@intel.com diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 0ea37d941..c645de60e 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -457,6 +457,18 @@ description: "Directory where the build artifact will be located upon the completion of the build." - parameter: + name: 'ericsson-build3-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-build3' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + +- parameter: name: 'huawei-build-defaults' parameters: - node: @@ -757,6 +769,21 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: + name: zte-virtual6-defaults + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - zte-virtual6 + default-slaves: + - zte-virtual6 + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + +- parameter: name: 'juniper-pod1-defaults' parameters: - node: @@ -931,6 +958,12 @@ default-slave: 'doctor-fuel-verify' - parameter: + name: 'doctor-fuel-build-aarch64-verify-defaults' + parameters: + - 'doctor-defaults': + default-slave: 'doctor-fuel-build-aarch64-verify' + +- parameter: name: 'doctor-joid-verify-defaults' parameters: - 'doctor-defaults': diff --git a/jjb/joid/joid-daily-jobs.yml b/jjb/joid/joid-daily-jobs.yml index 569bff293..947b2f1ab 100644 --- a/jjb/joid/joid-daily-jobs.yml +++ b/jjb/joid/joid-daily-jobs.yml @@ -18,6 +18,11 @@ branch: 'stable/{stream}' disabled: false gs-pathname: '/{stream}' + master: &master + stream: master + branch: '{stream}' + disabled: false + gs-pathname: '' # ------------------------------- # POD, INSTALLER, AND BRANCH MAPPING # ------------------------------- @@ -27,7 +32,9 @@ - baremetal: slave-label: joid-baremetal <<: *euphrates - + - baremetal: + slave-label: joid-baremetal + <<: *master # ------------------------------- # scenarios # ------------------------------- @@ -56,6 +63,14 @@ auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' - 'os-nosdn-openbaton-ha': auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' + - 'k8-canal-lb-noha': + auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' + - 'k8-nosdn-lb_ceph-noha': + auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' + - 'k8-multus-lb-noha': + auto-trigger-name: 'daily-trigger-disabled' + - 'os-ovn-nofeature-ha': + auto-trigger-name: 'daily-trigger-disabled' jobs: - 'joid-{scenario}-{pod}-daily-{stream}' @@ -222,53 +237,148 @@ name: 'joid-os-nosdn-nofeature-ha-baremetal-euphrates-trigger' triggers: - timed: '5 2 * * *' +# os-nosdn-nofeature-ha trigger - branch: master +- trigger: + name: 'joid-os-nosdn-nofeature-ha-baremetal-master-trigger' + triggers: + - timed: '5 2 * * *' # os-odl_l2-nofeature-ha trigger - branch: euphrates - trigger: name: 'joid-os-odl_l2-nofeature-ha-baremetal-euphrates-trigger' triggers: - timed: '5 7 * * *' +# os-odl_l2-nofeature-ha trigger - branch: master +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-baremetal-master-trigger' + triggers: + - timed: '5 7 * * *' # os-nosdn-lxd-noha trigger - branch: euphrates - trigger: name: 'joid-os-nosdn-lxd-noha-baremetal-euphrates-trigger' triggers: - timed: '5 22 * * *' +# os-nosdn-lxd-noha trigger - branch: master +- trigger: + name: 'joid-os-nosdn-lxd-noha-baremetal-master-trigger' + triggers: + - timed: '5 22 * * *' # os-nosdn-lxd-ha trigger - branch: euphrates - trigger: name: 'joid-os-nosdn-lxd-ha-baremetal-euphrates-trigger' triggers: - timed: '5 10 * * *' +# os-nosdn-lxd-ha trigger - branch: master +- trigger: + name: 'joid-os-nosdn-lxd-ha-baremetal-master-trigger' + triggers: + - timed: '5 10 * * *' # os-nosdn-nofeature-noha trigger - branch: euphrates - trigger: name: 'joid-os-nosdn-nofeature-noha-baremetal-euphrates-trigger' triggers: - timed: '5 4 * * *' +# os-nosdn-nofeature-noha trigger - branch: master +- trigger: + name: 'joid-os-nosdn-nofeature-noha-baremetal-master-trigger' + triggers: + - timed: '5 4 * * *' # k8-nosdn-nofeature-noha trigger - branch: euphrates - trigger: name: 'joid-k8-nosdn-nofeature-noha-baremetal-euphrates-trigger' triggers: - timed: '5 15 * * *' +# k8-nosdn-nofeature-noha trigger - branch: master +- trigger: + name: 'joid-k8-nosdn-nofeature-noha-baremetal-master-trigger' + triggers: + - timed: '5 15 * * *' # k8-nosdn-lb-noha trigger - branch: euphrates - trigger: name: 'joid-k8-nosdn-lb-noha-baremetal-euphrates-trigger' triggers: - timed: '5 20 * * *' +# k8-nosdn-lb-noha trigger - branch: master +- trigger: + name: 'joid-k8-nosdn-lb-noha-baremetal-master-trigger' + triggers: + - timed: '5 20 * * *' # k8-ovn-lb-noha trigger - branch: euphrates - trigger: name: 'joid-k8-ovn-lb-noha-baremetal-euphrates-trigger' triggers: - timed: '5 17 * * *' +# k8-ovn-lb-noha trigger - branch: master +- trigger: + name: 'joid-k8-ovn-lb-noha-baremetal-master-trigger' + triggers: + - timed: '5 17 * * *' # os-nosdn-openbaton-ha trigger - branch: euphrates - trigger: name: 'joid-os-nosdn-openbaton-ha-baremetal-euphrates-trigger' triggers: - - timed: '5 25 * * *' + - timed: '5 23 * * *' +# os-nosdn-openbaton-ha trigger - branch: master +- trigger: + name: 'joid-os-nosdn-openbaton-ha-baremetal-master-trigger' + triggers: + - timed: '5 23 * * *' # os-ocl-nofeature-ha trigger - branch: euphrates - trigger: name: 'joid-os-ocl-nofeature-ha-baremetal-euphrates-trigger' triggers: - timed: '5 13 * * *' -# os-ocl-nofeature-ha trigger - branch: euphrates +# os-ocl-nofeature-ha trigger - branch: master +- trigger: + name: 'joid-os-ocl-nofeature-ha-baremetal-master-trigger' + triggers: + - timed: '5 13 * * *' +# os-ocl-nofeature-noha trigger - branch: euphrates - trigger: name: 'joid-os-ocl-nofeature-noha-baremetal-euphrates-trigger' triggers: - - timed: '5 17 * * *' + - timed: '5 16 * * *' +# os-ocl-nofeature-noha trigger - branch: master +- trigger: + name: 'joid-os-ocl-nofeature-noha-baremetal-master-trigger' + triggers: + - timed: '5 16 * * *' +# k8-canal-lb-noha trigger - branch: euphrates +- trigger: + name: 'joid-k8-canal-lb-noha-baremetal-euphrates-trigger' + triggers: + - timed: '5 14 * * *' +# k8-canal-lb-noha trigger - branch: master +- trigger: + name: 'joid-k8-canal-lb-noha-baremetal-master-trigger' + triggers: + - timed: '5 14 * * *' +# k8-multus-lb-noha trigger - branch: euphrates +- trigger: + name: 'joid-k8-multus-lb-noha-baremetal-euphrates-trigger' + triggers: + - timed: '5 18 * * *' +# k8-multus-lb-noha trigger - branch: master +- trigger: + name: 'joid-k8-multus-lb-noha-baremetal-master-trigger' + triggers: + - timed: '5 18 * * *' +# k8-nosdn-lb_ceph-noha trigger - branch: euphrates +- trigger: + name: 'joid-k8-nosdn-lb_ceph-noha-baremetal-euphrates-trigger' + triggers: + - timed: '5 18 * * *' +# k8-nosdn-lb_ceph-noha trigger - branch: master +- trigger: + name: 'joid-k8-nosdn-lb_ceph-noha-baremetal-master-trigger' + triggers: + - timed: '5 18 * * *' +# os-ovn-nofeature-ha trigger - branch: euphrates +- trigger: + name: 'joid-os-ovn-nofeature-ha-baremetal-euphrates-trigger' + triggers: + - timed: '5 19 * * *' +# os-ovn-nofeature-ha trigger - branch: master +- trigger: + name: 'joid-os-ovn-nofeature-ha-baremetal-master-trigger' + triggers: + - timed: '5 19 * * *' diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml index aae6e028e..ad497e97d 100644 --- a/jjb/kvmfornfv/kvmfornfv.yml +++ b/jjb/kvmfornfv/kvmfornfv.yml @@ -8,7 +8,7 @@ - master: branch: '{stream}' gs-pathname: '' - disabled: true + disabled: false - euphrates: branch: 'stable/{stream}' gs-pathname: '/{stream}' diff --git a/jjb/opnfvdocs/docs-rtd.yaml b/jjb/opnfvdocs/docs-rtd.yaml index e15b09812..28f9354fa 100644 --- a/jjb/opnfvdocs/docs-rtd.yaml +++ b/jjb/opnfvdocs/docs-rtd.yaml @@ -38,6 +38,7 @@ files: 'docs/**/*.*' builders: + - 'remove-old-docs-from-opnfv-artifacts' - shell: | if [ $GERRIT_BRANCH == "master" ]; then RTD_BUILD_VERSION=latest @@ -91,4 +92,6 @@ pip install --upgrade pip pip freeze pip install tox + sed -i s,\-b\ html,\-b\ singlehtml,g tox.ini tox -edocs + - 'upload-review-docs' diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml index 95bc8a9dd..3dfb9d155 100644 --- a/jjb/opnfvdocs/opnfvdocs.yml +++ b/jjb/opnfvdocs/opnfvdocs.yml @@ -11,7 +11,6 @@ jobs: - 'opnfvdocs-verify-shellcheck-{stream}' - 'opnfvdocs-merge-shellcheck-{stream}' - - 'opnfvdocs-daily-{stream}' stream: - master: @@ -109,31 +108,3 @@ builders: - check-bash-syntax - -- job-template: - name: 'opnfvdocs-daily-{stream}' - - disabled: true - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - string: - name: GS_URL - default: '$GS_BASE{gs-pathname}' - description: "Directory where the build artifact will be located upon the completion of the build." - - string: - name: GIT_CLONE_BASE - default: ssh://gerrit.opnfv.org:29418 - description: "Used for overriding the GIT URL coming from parameters macro." - - scm: - - git-scm - - triggers: - - timed: '0 H/6 * * *' - - builders: - - build-html-and-pdf-docs-output -# - upload-generated-docs-to-opnfv-artifacts diff --git a/jjb/ovn4nfv/ovn4nfv-project-jobs.yml b/jjb/ovn4nfv/ovn4nfv-project-jobs.yml index b5e078ca6..97d1baefd 100644 --- a/jjb/ovn4nfv/ovn4nfv-project-jobs.yml +++ b/jjb/ovn4nfv/ovn4nfv-project-jobs.yml @@ -4,7 +4,6 @@ project: '{name}' - stream: - master: branch: '{stream}' @@ -33,10 +32,11 @@ max-per-node: 1 option: 'project' - parametert: + parameters: - project-parameter: project: '{project}' branch: '{branch}' + - opnfv-build-defaults scm: - git-scm diff --git a/jjb/pharos/check-jinja2.yml b/jjb/pharos/check-jinja2.yml new file mode 100644 index 000000000..cfc123e4c --- /dev/null +++ b/jjb/pharos/check-jinja2.yml @@ -0,0 +1,85 @@ +--- +######################## +# Job configuration to validate jinja2 files +######################## +- project: + + name: validate-pdf-templates + + project: 'pharos' + + jobs: + - 'validate-pdf-jinja2-templates-{stream}' + + stream: + - master: + branch: '{stream}' + disabled: false + - euphrates: + branch: 'stable/{stream}' + disabled: false + +######################## +# job templates +######################## + +- job-template: + name: 'validate-pdf-jinja2-templates-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + branch: '{branch}' + - node: + name: SLAVE_NAME + description: Slave to execute jnija template test + default-slaves: + - lf-build1 + allowed-multiselect: true + ignore-offline-nodes: true + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*.yaml' + - compare-type: ANT + pattern: 'check-jinja2.sh' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + + builders: + - check-pdf-jinja + +- builder: + name: check-pdf-jinja + builders: + - shell: | + $WORKSPACE/config/utils/check-jinja2.sh diff --git a/jjb/qtip/qtip-experimental-jobs.yml b/jjb/qtip/qtip-experimental-jobs.yml index fdfed2601..103069fb7 100644 --- a/jjb/qtip/qtip-experimental-jobs.yml +++ b/jjb/qtip/qtip-experimental-jobs.yml @@ -7,7 +7,7 @@ name: qtip-experimental-jobs project: qtip jobs: - - 'qtip-experimental-{stream}' + - 'qtip-experimental-{pod}-{stream}' stream: - master: branch: '{stream}' @@ -18,12 +18,17 @@ gs-pathname: '/{stream}' disabled: false + pod: + - zte-virtual6: + installer: fuel + pod: zte-virtual6 + ################################ ## job templates ################################# - job-template: - name: 'qtip-experimental-{stream}' + name: 'qtip-experimental-{pod}-{stream}' disabled: '{obj:disabled}' @@ -31,9 +36,8 @@ - project-parameter: project: '{project}' branch: '{branch}' - # Pin the tests on zte-pod4 with apex deployment - - apex-defaults - - zte-virtual5-defaults + - '{installer}-defaults' + - '{pod}-defaults' scm: - git-scm-gerrit diff --git a/jjb/qtip/qtip-validate-jobs.yml b/jjb/qtip/qtip-validate-jobs.yml index febe28619..d9d3e72e6 100644 --- a/jjb/qtip/qtip-validate-jobs.yml +++ b/jjb/qtip/qtip-validate-jobs.yml @@ -40,18 +40,22 @@ installer: apex pod: zte-virtual5 <<: *euphrates + - compute: + installer: fuel + pod: zte-virtual6 + <<: *master # ------------------------------- # JOB LIST # ------------------------------- jobs: - - 'qtip-{qpi}-{installer}-{stream}' + - 'qtip-{qpi}-{installer}-{pod}-{stream}' ################################ # job templates ################################ - job-template: - name: 'qtip-{qpi}-{installer}-{stream}' + name: 'qtip-{qpi}-{installer}-{pod}-{stream}' disabled: false parameters: @@ -85,7 +89,7 @@ wrappers: - timeout: - timeout: 15 + timeout: 30 abort: true - timestamps diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml index de7fcd6c0..afeb35821 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/automate.yml @@ -19,7 +19,7 @@ - '{module}-automate-{phase}-{stream}' - '{module}-verify-{stream}' - project: 'releng' + project: 'releng-testresults' - job: name: 'testapi-mongodb-backup' @@ -30,11 +30,11 @@ default: 'testresults' description: 'Slave label on Jenkins' - project-parameter: - project: 'releng' + project: 'releng-testresults' branch: 'master' - string: name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/releng + default: https://gerrit.opnfv.org/gerrit/releng-testresults description: 'Git URL to use on this Jenkins Slave' scm: @@ -79,11 +79,11 @@ branch-pattern: '**/{branch}' file-paths: - compare-type: 'ANT' - pattern: 'utils/test/{module}/**' + pattern: '{module}/**' builders: - shell: | - cd ./utils/test/{module}/ + cd {module}/ tox if [ -e *.xml ];then cp *.xml $WORKSPACE @@ -143,7 +143,7 @@ branch-pattern: '**/{branch}' file-paths: - compare-type: 'ANT' - pattern: 'utils/test/{module}/**' + pattern: '{module}/**' builders: - description-setter: @@ -208,8 +208,8 @@ - builder: name: 'docker-update' builders: - - shell: - !include-raw: ./docker-update.sh + - shell: | + bash ./ci/docker-update.sh - builder: name: 'testapi-automate-generate-doc-macro' @@ -221,13 +221,13 @@ name: 'testapi-doc-build' builders: - shell: | - bash ./jjb/releng/htmlize/doc-build.sh + bash ./ci/htmlize/doc-build.sh - builder: name: 'upload-doc-artifact' builders: - shell: | - bash ./jjb/releng/htmlize/push-doc-artifact.sh + bash ./ci/htmlize/push-doc-artifacts.sh - builder: name: 'reporting-automate-generate-doc-macro' @@ -238,7 +238,7 @@ name: 'testapi-automate-docker-deploy-macro' builders: - shell: | - sudo bash ./jjb/releng/docker-deploy.sh "sudo docker run -dti --name testapi -p 8082:8000 + sudo bash ./ci/docker-deploy.sh "sudo docker run -dti --name testapi -p 8082:8000 -e mongodb_url=mongodb://172.17.0.1:27017 -e base_url=http://testresults.opnfv.org/test opnfv/testapi" \ "http://testresults.opnfv.org/test/" "testapi" @@ -247,14 +247,15 @@ name: 'reporting-automate-docker-deploy-macro' builders: - shell: | - sudo bash ./jjb/releng/docker-deploy.sh "sudo docker run -itd --name reporting -p 8084:8000 opnfv/reporting" \ + sudo bash ./ci/docker-deploy.sh \ + "sudo docker run -itd --name reporting -p 8084:8000 opnfv/reporting" \ "http://testresults.opnfv.org/reporting/index.html" "reporting" - builder: name: mongodb-backup builders: - - shell: | - bash ./jjb/releng/testapi-backup-mongodb.sh + - shell: + bash ./ci/testapi-backup-mongodb.sh ################################ # job publishers diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml new file mode 100644 index 000000000..299908d66 --- /dev/null +++ b/jjb/releng/compass4nfv-docker.yml @@ -0,0 +1,177 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: compass-docker + + project: compass-containers + + stream: + - master: + branch: '{stream}' + disabled: false + - euphrates: + branch: 'stable/{stream}' + disabled: true + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + + # yamllint disable rule:key-duplicates + image: + - 'tasks' + - 'cobbler' + - 'db' + - 'deck' + - 'tasks-k8s' + - 'tasks-osa' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + RELEASE_VERSION=$RELEASE_VERSION + DOCKERFILE=$DOCKERFILE + kill-phase-on: FAILURE + abort-all-jobs: false + + # yamllint enable rule:key-duplicates + jobs: + - "compass-docker-{stream}" + - "compass-{image}-build-{arch_tag}-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'compass-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - compass-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/30 * * * *" + + builders: + - multijob: + name: 'build compass-tasks images' + execution-type: PARALLEL + projects: + - name: 'compass-tasks-build-amd64-{stream}' + <<: *build-job-settings + - multijob: + name: 'build all compass images' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'compass-cobbler-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-db-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-deck-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-tasks-k8s-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-tasks-osa-build-amd64-{stream}' + <<: *build-job-settings + + publishers: + - 'compass-amd64-recipients' + +- job-template: + name: 'compass-{image}-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - compass-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + - string: + name: DOCKER_REPO_NAME + default: "opnfv/compass-{image}" + description: "Dockerhub repo to be pushed to." + - string: + name: DOCKER_DIR + default: "compass-{image}" + description: "Directory containing files needed by the Dockerfile" + scm: + - git-scm + builders: + - shell: + !include-raw-escape: ./opnfv-docker.sh + +# parameter macro +- parameter: + name: compass-job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: RELEASE_VERSION + default: "" + description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + - string: + name: DOCKERFILE + default: "Dockerfile" + description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "" + description: "If set, this value will be added to the docker image tag as a prefix" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + +# publisher macros +- publisher: + name: 'compass-amd64-recipients' + publishers: + - email: + recipients: > + chigang@huawei.com + huangxiangyui5@huawei.com + xueyifei@huawei.com + wutianwei1@huawei.com diff --git a/jjb/releng/docker-deploy.sh b/jjb/releng/docker-deploy.sh deleted file mode 100644 index 1e8357717..000000000 --- a/jjb/releng/docker-deploy.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one * -# or more contributor license agreements. See the NOTICE file * -# distributed with this work for additional information * -# regarding copyright ownership. The ASF licenses this file * -# to you under the Apache License, Version 2.0 (the * -# "License"); you may not use this file except in compliance * -# with the License. You may obtain a copy of the License at * -# * -# http://www.apache.org/licenses/LICENSE-2.0 * -# * -# Unless required by applicable law or agreed to in writing, * -# software distributed under the License is distributed on an * -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * -# KIND, either express or implied. See the License for the * -# specific language governing permissions and limitations * -# under the License. * - - -command=$1 -url=$2 -module=$3 - -REPO="opnfv" -latest_image=$REPO/$module:latest -old_image=$REPO/$module:old -latest_container_name=$module -old_container_name=$module"_old" -latest_container_id= -old_container_id= -new_start_container= - -function DEBUG() { - echo `date "+%Y-%m-%d %H:%M:%S.%N"` ": $1" -} - -function check_connectivity() { - # check update status via test the connectivity of provide url - sleep 5 - cmd=`curl -s --head --request GET ${url} | grep '200 OK' > /dev/null` - rc=$? - DEBUG $rc - if [[ $rc == 0 ]]; then - return 0 - else - return 1 - fi -} - - -function pull_latest_image() { - DEBUG "pull latest image $latest_image" - docker pull $latest_image -} - -function get_latest_running_container() { - latest_container_id=`docker ps -q --filter name=^/$latest_container_name$` -} - -function get_old_running_container() { - old_container_id=`docker ps -q --filter name=^/$old_container_name$` -} - -function delete_old_image() { - DEBUG "delete old image: $old_image" - docker rmi -f $old_image -} - -function delete_old_container() { - DEBUG "delete old container: $old_container_name" - docker ps -a -q --filter name=^/$old_container_name$ | xargs docker rm -f &>/dev/null -} - -function delete_latest_container() { - DEBUG "delete latest container: $module" - docker ps -a -q --filter name=^/$latest_container_name$ | xargs docker rm -f &>/dev/null -} - -function delete_latest_image() { - DEBUG "delete latest image: $REPO/$module:latest" - docker rmi -f $latest_image -} - -function change_image_tag_2_old() { - DEBUG "change image tag 2 old" - docker tag $latest_image $old_image - docker rmi -f $latest_image -} - -function mark_latest_container_2_old() { - DEBUG "mark latest container to be old" - docker rename "$latest_container_name" "$old_container_name" -} - -function stop_old_container() { - DEBUG "stop old container" - docker stop "$old_container_name" -} - -function run_latest_image() { - new_start_container=`$command` - DEBUG "run latest image: $new_start_container" -} - -get_latest_running_container -get_old_running_container - -if [[ ! -z $latest_container_id ]]; then - DEBUG "latest container is running: $latest_container_id" - delete_old_container - delete_old_image - change_image_tag_2_old - mark_latest_container_2_old - pull_latest_image - stop_old_container - run_latest_image - -elif [[ ! -z $old_container_id ]]; then - DEBUG "old container is running: $old_container_id" - delete_latest_container - delete_latest_image - pull_latest_image - stop_old_container - run_latest_image -else - DEBUG "no container is running" - delete_old_container - delete_old_image - delete_latest_container - delete_latest_image - pull_latest_image - run_latest_image -fi - -if check_connectivity; then - DEBUG "CONGRATS: $module update successfully" -else - DEBUG "ATTENTION: $module update failed" - id=`docker ps -a -q --filter name=^/$old_container_name$` - if [[ ! -z $id ]]; then - DEBUG "start old container instead" - docker stop $new_start_container - docker start $id - fi - if ! check_connectivity; then - DEBUG "BIG ISSUE: no container is running normally" - fi - exit 1 -fi - -docker images -docker ps -a diff --git a/jjb/releng/docker-update.sh b/jjb/releng/docker-update.sh deleted file mode 100644 index 559ac835c..000000000 --- a/jjb/releng/docker-update.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one * -# or more contributor license agreements. See the NOTICE file * -# distributed with this work for additional information * -# regarding copyright ownership. The ASF licenses this file * -# to you under the Apache License, Version 2.0 (the * -# "License"); you may not use this file except in compliance * -# with the License. You may obtain a copy of the License at * -# * -# http://www.apache.org/licenses/LICENSE-2.0 * -# * -# Unless required by applicable law or agreed to in writing, * -# software distributed under the License is distributed on an * -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * -# KIND, either express or implied. See the License for the * -# specific language governing permissions and limitations * -# under the License. * - -set -o errexit -set -o nounset - -cd $WORKSPACE/utils/test/$MODULE_NAME/docker/ - -# Remove previous containers -docker ps -a | grep "opnfv/$MODULE_NAME" | awk '{ print $1 }' | xargs -r docker rm -f - -# Remove previous images -docker images | grep "opnfv/$MODULE_NAME" | awk '{ print $3 }' | xargs -r docker rmi -f - -# Start build -docker build --no-cache -t opnfv/$MODULE_NAME:$DOCKER_TAG . - -# Push Image -docker push opnfv/$MODULE_NAME:$DOCKER_TAG diff --git a/jjb/releng/functest-docker.yml b/jjb/releng/functest-docker.yml new file mode 100644 index 000000000..acf381f88 --- /dev/null +++ b/jjb/releng/functest-docker.yml @@ -0,0 +1,289 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: functest-docker + + project: functest + + stream: + - master: + branch: '{stream}' + disabled: false + - euphrates: + branch: 'stable/{stream}' + disabled: false + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + - 'arm64': + slave_label: 'opnfv-build-ubuntu-arm' + + # yamllint disable rule:key-duplicates + image: + - 'core' + - 'healthcheck' + - 'features' + - 'components' + - 'parser' + - 'smoke' + - 'vnf' + - 'restapi' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + RELEASE_VERSION=$RELEASE_VERSION + DOCKERFILE=$DOCKERFILE + kill-phase-on: FAILURE + abort-all-jobs: false + + manifest-job-settings: &manifest-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: + RELEASE_VERSION=$RELEASE_VERSION + kill-phase-on: FAILURE + abort-all-jobs: false + + # yamllint enable rule:key-duplicates + jobs: + - "functest-docker-{stream}" + - "functest-{image}-build-{arch_tag}-{stream}" + - "functest-{image}-manifest-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'functest-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/30 * * * *" + + builders: + - multijob: + name: 'build functest-core images' + execution-type: PARALLEL + projects: + - name: 'functest-core-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-core-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish functest-core manifests' + execution-type: PARALLEL + projects: + - name: 'functest-core-manifest-{stream}' + <<: *manifest-job-settings + - multijob: + name: 'build all functest images' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'functest-healthcheck-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-healthcheck-build-arm64-{stream}' + <<: *build-job-settings + - name: 'functest-features-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-features-build-arm64-{stream}' + <<: *build-job-settings + - name: 'functest-components-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-components-build-arm64-{stream}' + <<: *build-job-settings + - name: 'functest-parser-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-parser-build-arm64-{stream}' + <<: *build-job-settings + - name: 'functest-smoke-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-smoke-build-arm64-{stream}' + <<: *build-job-settings + - name: 'functest-vnf-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-restapi-build-amd64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish all manifests' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'functest-healthcheck-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-features-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-components-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-parser-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-smoke-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-vnf-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-restapi-manifest-{stream}' + <<: *manifest-job-settings + + publishers: + - 'functest-amd64-recipients' + - 'functest-arm64-recipients' + +- job-template: + name: 'functest-{image}-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + scm: + - git-scm + builders: + - shell: | + #!/bin/bash -ex + case "{arch_tag}" in + "arm64") + sudo amd64_dirs= arm64_dirs=docker/{image} bash ./build.sh ;; + *) + sudo amd64_dirs=docker/{image} arm64_dirs= bash ./build.sh ;; + esac + exit $? + +- job-template: + name: 'functest-{image}-manifest-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'opnfv-build-ubuntu' + description: 'Slave label on Jenkins' + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: RELEASE_VERSION + default: "" + description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + disabled: '{obj:disabled}' + + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + case "{image}" in + "vnf"|"restapi") + sudo manifest-tool push from-args \ + --platforms linux/amd64 \ + --template opnfv/functest-{image}:ARCH-$tag \ + --target opnfv/functest-{image}:$tag ;; + *) + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template opnfv/functest-{image}:ARCH-$tag \ + --target opnfv/functest-{image}:$tag ;; + esac + exit $? + +# parameter macro +- parameter: + name: job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: RELEASE_VERSION + default: "" + description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + - string: + name: DOCKERFILE + default: "Dockerfile" + description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "{arch_tag}" + description: "If set, this value will be added to the docker image tag as a prefix" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + +# publisher macros +- publisher: + name: 'functest-arm64-recipients' + publishers: + - email: + recipients: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + delia.popescu@enea.com + +- publisher: + name: 'functest-amd64-recipients' + publishers: + - email: + recipients: > + jalausuch@suse.com morgan.richomme@orange.com + cedric.ollivier@orange.com feng.xiaowei@zte.com.cn + juha.kosonen@nokia.com wangwulin@huawei.com diff --git a/jjb/releng/htmlize/doc-build.sh b/jjb/releng/htmlize/doc-build.sh deleted file mode 100644 index b89baa8e6..000000000 --- a/jjb/releng/htmlize/doc-build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one * -# or more contributor license agreements. See the NOTICE file * -# distributed with this work for additional information * -# regarding copyright ownership. The ASF licenses this file * -# to you under the Apache License, Version 2.0 (the * -# "License"); you may not use this file except in compliance * -# with the License. You may obtain a copy of the License at * -# * -# http://www.apache.org/licenses/LICENSE-2.0 * -# * -# Unless required by applicable law or agreed to in writing, * -# software distributed under the License is distributed on an * -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * -# KIND, either express or implied. See the License for the * -# specific language governing permissions and limitations * -# under the License. * - -set -o errexit - -# Create virtual environment -virtualenv $WORKSPACE/testapi_venv -source $WORKSPACE/testapi_venv/bin/activate - -# Swgger Codegen Tool -url="http://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/2.2.1/swagger-codegen-cli-2.2.1.jar" - -# Check for jar file locally and in the repo -if [ ! -f swagger-codegen-cli.jar ]; -then - wget http://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/2.2.1/swagger-codegen-cli-2.2.1.jar -O swagger-codegen-cli.jar -fi - -# Install Pre-requistics -pip install requests - -python ./jjb/releng/htmlize/htmlize.py -o ${WORKSPACE}/ diff --git a/jjb/releng/htmlize/htmlize.py b/jjb/releng/htmlize/htmlize.py deleted file mode 100644 index da6a6cf91..000000000 --- a/jjb/releng/htmlize/htmlize.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python - -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 - -import argparse -import requests -import json -import os - - -def main(args): - - # Merging two specs - api_response = requests.get(args.api_declaration_url) - api_response = json.loads(api_response.content) - resource_response = requests.get(args.resource_listing_url) - resource_response = json.loads(resource_response.content) - resource_response['models'] = api_response['models'] - resource_response['apis'] = api_response['apis'] - - # Storing the swagger specs - with open('specs.json', 'w') as outfile: - json.dump(resource_response, outfile) - - # Generating html page - cmd = 'java -jar swagger-codegen-cli.jar generate \ - -i specs.json -l html2 -o %s' % (args.output_directory) - if os.system(cmd) == 0: - exit(0) - else: - exit(1) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Create \ - Swagger Spec documentation') - parser.add_argument('-ru', '--resource-listing-url', - type=str, - required=False, - default=('http://testresults.opnfv.org' - '/test/swagger/resources.json'), - help='Resource Listing Spec File') - parser.add_argument('-au', '--api-declaration-url', - type=str, - required=False, - default=('http://testresults.opnfv.org' - '/test/swagger/APIs'), - help='API Declaration Spec File') - parser.add_argument('-o', '--output-directory', - required=True, - default='./', - help='Output Directory where the \ - file should be stored') - main(parser.parse_args()) diff --git a/jjb/releng/htmlize/push-doc-artifact.sh b/jjb/releng/htmlize/push-doc-artifact.sh deleted file mode 100644 index 4cf1988b0..000000000 --- a/jjb/releng/htmlize/push-doc-artifact.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ - -project=$PROJECT -workspace=$WORKSPACE -artifact_dir="$project/docs" - -set +e -gsutil&>/dev/null -if [ $? != 0 ]; then - echo "Not possible to push results to artifact: gsutil not installed" - exit 1 -else - gsutil ls gs://artifacts.opnfv.org/"$project"/ &>/dev/null - if [ $? != 0 ]; then - echo "Not possible to push results to artifact: gsutil not installed." - exit 1 - else - echo "Uploading document to artifact $artifact_dir" - gsutil cp "$workspace"/index.html gs://artifacts.opnfv.org/"$artifact_dir"/testapi.html >/dev/null 2>&1 - echo "Document can be found at http://artifacts.opnfv.org/releng/docs/testapi.html" - fi -fi diff --git a/jjb/releng/opnfv-docker-arm.yml b/jjb/releng/opnfv-docker-arm.yml index b488592db..d1566b48d 100644 --- a/jjb/releng/opnfv-docker-arm.yml +++ b/jjb/releng/opnfv-docker-arm.yml @@ -15,11 +15,6 @@ stream: euphrates branch: 'stable/{stream}' disabled: false - functest-arm-receivers: &functest-arm-receivers - receivers: > - cristina.pauna@enea.com - alexandru.avadanii@enea.com - delia.popescu@enea.com dovetail-arm-receivers: &dovetail-arm-receivers receivers: > cristina.pauna@enea.com @@ -40,7 +35,7 @@ other-receivers: &other-receivers receivers: '' - dockerfile: "Dockerfile.aarch64" + dockerfile: "Dockerfile" dockerdir: "docker" docker_repo_name: "opnfv/{project}_aarch64" arch_tag: "" @@ -48,10 +43,6 @@ # yamllint disable rule:key-duplicates dockerrepo: # projects with jobs for master - - 'functest': - project: 'functest' - <<: *master - <<: *functest-arm-receivers - 'dovetail': project: 'dovetail' <<: *master @@ -59,7 +50,6 @@ - 'storperf-master': project: 'storperf' dockerdir: 'docker/storperf-master' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-master' arch_tag: 'aarch64' <<: *master @@ -67,7 +57,6 @@ - 'storperf-graphite': project: 'storperf' dockerdir: 'docker/storperf-graphite' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-graphite' arch_tag: 'aarch64' <<: *master @@ -75,7 +64,6 @@ - 'storperf-httpfrontend': project: 'storperf' dockerdir: 'docker/storperf-httpfrontend' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-httpfrontend' arch_tag: 'aarch64' <<: *master @@ -83,7 +71,6 @@ - 'storperf-reporting': project: 'storperf' dockerdir: 'docker/storperf-reporting' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-reporting' arch_tag: 'aarch64' <<: *master @@ -91,7 +78,6 @@ - 'storperf-swaggerui': project: 'storperf' dockerdir: 'docker/storperf-swaggerui' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-swaggerui' arch_tag: 'aarch64' <<: *master @@ -101,10 +87,6 @@ <<: *master <<: *yardstick-arm-receivers # projects with jobs for stable/euphrates - - 'functest': - project: 'functest' - <<: *euphrates - <<: *functest-arm-receivers - 'dovetail': project: 'dovetail' <<: *euphrates @@ -112,7 +94,6 @@ - 'storperf-master': project: 'storperf' dockerdir: 'docker/storperf-master' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-master' arch_tag: 'aarch64' <<: *euphrates @@ -120,7 +101,6 @@ - 'storperf-graphite': project: 'storperf' dockerdir: 'docker/storperf-graphite' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-graphite' arch_tag: 'aarch64' <<: *euphrates @@ -128,7 +108,6 @@ - 'storperf-httpfrontend': project: 'storperf' dockerdir: 'docker/storperf-httpfrontend' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-httpfrontend' arch_tag: 'aarch64' <<: *euphrates @@ -136,7 +115,6 @@ - 'storperf-reporting': project: 'storperf' dockerdir: 'docker/storperf-reporting' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-reporting' arch_tag: 'aarch64' <<: *euphrates @@ -144,7 +122,6 @@ - 'storperf-swaggerui': project: 'storperf' dockerdir: 'docker/storperf-swaggerui' - dockerfile: 'Dockerfile' docker_repo_name: 'opnfv/storperf-swaggerui' arch_tag: 'aarch64' <<: *euphrates diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index 402099a91..ec7b3fda3 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -17,6 +17,36 @@ echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..." echo "--------------------------------------------------------" echo +function remove_containers_images() +{ + # Remove previous running containers if exist + if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then + echo "Removing existing $DOCKER_REPO_NAME containers..." + docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f + t=60 + # Wait max 60 sec for containers to be removed + while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do + sleep 1 + let t=t-1 + done + fi + + + # Remove existing images if exist + if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep $DOCKER_REPO_NAME + image_ids=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $3}')) + for id in "${image_ids[@]}"; do + if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $id)" ]]; then + echo "Removing docker image $DOCKER_REPO_NAME:$id..." + docker rmi -f $id + fi + done + fi +} + + count=30 # docker build jobs might take up to ~30 min while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep` ]]; do echo "Build or cleanup of $DOCKER_REPO_NAME in progress. Waiting..." @@ -28,46 +58,16 @@ while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep fi done -# Remove previous running containers if exist -if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then - echo "Removing existing $DOCKER_REPO_NAME containers..." - docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f - t=60 - # Wait max 60 sec for containers to be removed - while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do - sleep 1 - let t=t-1 - done -fi - +# Remove the existing containers and images before building +remove_containers_images -# Remove existing images if exist -if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep $DOCKER_REPO_NAME - image_ids=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $3}')) - for id in "${image_ids[@]}"; do - if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $id)" ]]; then - echo "Removing docker image $DOCKER_REPO_NAME:$id..." - docker rmi -f $id - fi - done -fi - -cd $WORKSPACE/$DOCKER_DIR -HOST_ARCH=$(uname -m) -if [ ! -f "${DOCKERFILE}" ]; then - # If this is expected to be a Dockerfile for other arch than x86 - # and it does not exist, but there is a patch for the said arch, - # then apply the patch and create the Dockerfile.${HOST_ARCH} file - if [[ "${DOCKERFILE}" == *"${HOST_ARCH}" && \ - -f "Dockerfile.${HOST_ARCH}.patch" ]]; then - patch -o Dockerfile."${HOST_ARCH}" Dockerfile \ - Dockerfile."${HOST_ARCH}".patch - else - echo "ERROR: No Dockerfile or ${HOST_ARCH} patch found." - exit 1 - fi +cd "$WORKSPACE/$DOCKER_DIR" || exit 1 +HOST_ARCH="$(uname -m)" +#If there is a patch for other arch then x86, apply the patch and +#replace Dockerfile file +dockerfile_patch="Dockerfile.${HOST_ARCH}.patch" +if [[ -f "${dockerfile_patch}" ]]; then + patch -f Dockerfile -p1 < "${dockerfile_patch}" fi # Get tag version @@ -107,7 +107,6 @@ echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" echo "--------------------------------------------------------" echo cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH - $ARCH_BUILD_ARG -f $DOCKERFILE ." echo ${cmd} @@ -125,3 +124,6 @@ if [[ "$PUSH_IMAGE" == "true" ]]; then echo docker push $DOCKER_REPO_NAME:$DOCKER_TAG fi + +# Remove the existing containers and images after building +remove_containers_images diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 862e77ac8..1a2756218 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -19,11 +19,6 @@ stream: euphrates branch: 'stable/{stream}' disabled: false - functest-receivers: &functest-receivers - receivers: > - jalausuch@suse.com morgan.richomme@orange.com - cedric.ollivier@orange.com feng.xiaowei@zte.com.cn - juha.kosonen@nokia.com wangwulin@huawei.com storperf-receivers: &storperf-receivers receivers: > mark.beierl@emc.com @@ -44,6 +39,10 @@ project: 'releng-anteater' <<: *master <<: *other-receivers + - 'barometer': + project: 'barometer' + <<: *master + <<: *other-receivers - 'bottlenecks': project: 'bottlenecks' <<: *master @@ -56,10 +55,6 @@ project: 'dovetail' <<: *master <<: *other-receivers - - 'functest': - project: 'functest' - <<: *master - <<: *functest-receivers - 'nfvbench': project: 'nfvbench' <<: *master @@ -102,24 +97,19 @@ project: 'yardstick' <<: *master <<: *other-receivers - # projects with jobs for Danube - - 'qtip': - project: 'qtip' - <<: *euphrates - <<: *other-receivers - - 'yardstick': - project: 'yardstick' - <<: *euphrates - <<: *other-receivers # projects with jobs for euphrates - 'bottlenecks': project: 'bottlenecks' <<: *euphrates <<: *other-receivers - - 'functest': - project: 'functest' + - 'nfvbench': + project: 'nfvbench' + <<: *euphrates + <<: *other-receivers + - 'qtip': + project: 'qtip' <<: *euphrates - <<: *functest-receivers + <<: *other-receivers - 'storperf-master': project: 'storperf' dockerdir: 'docker/storperf-master' @@ -150,8 +140,8 @@ arch_tag: 'x86_64' <<: *euphrates <<: *storperf-receivers - - 'nfvbench': - project: 'nfvbench' + - 'yardstick': + project: 'yardstick' <<: *euphrates <<: *other-receivers diff --git a/jjb/releng/opnfv-docs.yml b/jjb/releng/opnfv-docs.yml deleted file mode 100644 index 740274518..000000000 --- a/jjb/releng/opnfv-docs.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- -######################## -# Job configuration for opnfv-docs -######################## -- project: - - name: opnfv-docs - - project: opnfv-docs - - jobs: - - 'opnfv-docs-verify-{stream}' - - 'opnfv-docs-merge-{stream}' - - stream: - - master: - branch: '{stream}' - doc-version: '' - gs-pathname: '' - disabled: false - - euphrates: - branch: 'stable/{stream}' - doc-version: '5.0.0' - gs-pathname: '/{stream}/{doc-version}' - disabled: false - -######################## -# job templates -######################## - -- job-template: - name: 'opnfv-docs-verify-{stream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: $GERRIT_PROJECT - branch: '{branch}' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '*' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: docs/** - - compare-type: ANT - pattern: yardstick/** - - builders: - - upload-review-docs - -- job-template: - name: 'opnfv-docs-merge-{stream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: $GERRIT_PROJECT - branch: '{branch}' - - string: - name: GS_URL - default: '$GS_BASE{gs-pathname}' - description: "Directory where the build artifact will be located upon the completion of the build." - - string: - name: GERRIT_REFSPEC - default: 'refs/heads/{branch}' - description: "JJB configured GERRIT_REFSPEC parameter" - - scm: - - git-scm - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - - comment-added-contains-event: - comment-contains-value: 'rebuild docs' - projects: - - project-compare-type: 'ANT' - project-pattern: '*' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: docs/** - - compare-type: ANT - pattern: yardstick/** - - builders: - - upload-merged-docs diff --git a/jjb/releng/opnfv-repo-archiver.sh b/jjb/releng/opnfv-repo-archiver.sh index c9fdba379..2408c2a83 100644 --- a/jjb/releng/opnfv-repo-archiver.sh +++ b/jjb/releng/opnfv-repo-archiver.sh @@ -35,7 +35,7 @@ for PROJECT in "${PROJECT_LIST[@]}"; do # Don't license scan kernel or qemu in kvmfornfv if [ "$PROJECT" == "kvmfornfv" ]; then - rm -rf "$CLONE_PATH/$PROJECT/{kernel,qemu}" + rm -rf "$CLONE_PATH/$PROJECT/"{kernel,qemu} fi done diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index d4f39708e..c4b7c8db3 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -5,7 +5,6 @@ - 'releng-verify-jjb' - 'releng-merge-jjb' - 'releng-comment-jjb' - - 'releng-generate-artifacts-api' project: 'releng' @@ -119,22 +118,3 @@ cd /opt/jenkins-ci/releng git pull jenkins-jobs update -r --delete-old jjb/ - -- job-template: - name: 'releng-generate-artifacts-api' - - # Generate and upload the JSON file to used for artifacts site - - parameters: - - project-parameter: - project: '{project}' - branch: 'master' - - scm: - - git-scm - - triggers: - - timed: '@hourly' - - builders: - - build-and-upload-artifacts-json-api diff --git a/jjb/releng/testapi-backup-mongodb.sh b/jjb/releng/testapi-backup-mongodb.sh deleted file mode 100644 index 795e479d9..000000000 --- a/jjb/releng/testapi-backup-mongodb.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -e - -# Run MongoDB backup -python $WORKSPACE/utils/test/testapi/update/templates/backup_mongodb.py -o $WORKSPACE/ - -# Compressing the dump -now=$(date +"%m_%d_%Y_%H_%M_%S") -echo $now - -file_name="testapi_mongodb_"$now".tar.gz" -echo $file_name - -tar cvfz "$file_name" test_results_collection* - -rm -rf test_results_collection* - -artifact_dir="testapibackup" -workspace="$WORKSPACE" - -set +e -/usr/local/bin/gsutil &>/dev/null -if [ $? != 0 ]; then - echo "Not possible to push results to artifact: gsutil not installed" - exit 1 -else - echo "Uploading mongodump to artifact $artifact_dir" - /usr/local/bin/gsutil cp -r "$workspace"/"$file_name" gs://artifacts.opnfv.org/"$artifact_dir"/ - echo "MongoDump can be found at http://artifacts.opnfv.org/$artifact_dir.html" -fi diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml index f8d39b2fb..bbce9748a 100644 --- a/jjb/xci/bifrost-verify-jobs.yml +++ b/jjb/xci/bifrost-verify-jobs.yml @@ -70,7 +70,9 @@ - build-blocker: use-build-blocker: true blocking-jobs: + - 'xci-verify-{distro}-.*' - '.*-bifrost-verify.*-{type}' + block-level: 'NODE' parameters: @@ -78,9 +80,6 @@ name: PROJECT_REPO default: 'https://git.openstack.org/openstack/bifrost' - string: - name: CLONE_LOCATION - default: '$WORKSPACE/bifrost' - - string: name: DISTRO default: '{distro}' - string: diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh index a86ba91c3..263f544ad 100755 --- a/jjb/xci/bifrost-verify.sh +++ b/jjb/xci/bifrost-verify.sh @@ -11,15 +11,13 @@ set -o errexit set -o nounset set -o pipefail -git clone https://git.openstack.org/openstack/bifrost $WORKSPACE/bifrost git clone https://gerrit.opnfv.org/gerrit/releng-xci $WORKSPACE/releng-xci -# checkout the patch -cd $CLONE_LOCATION +cd $WORKSPACE git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD # combine opnfv and upstream scripts/playbooks -/bin/cp -rf $WORKSPACE/releng-xci/bifrost/* $WORKSPACE/bifrost/ +/bin/cp -rf $WORKSPACE/releng-xci/bifrost/* $WORKSPACE/ cd $WORKSPACE/releng-xci cat > bifrost_test.sh<<EOF @@ -41,8 +39,11 @@ case ${DISTRO} in *suse*) VM_DISTRO=opensuse ;; esac +export XCI_BUILD_CLEAN_VM_OS=false +export XCI_UPDATE_CLEAN_VM_OS=true + ./xci/scripts/vm/start-new-vm.sh $VM_DISTRO -rsync -a $WORKSPACE/releng-xci ${VM_DISTRO}_xci_vm:~/bifrost +rsync -a $WORKSPACE/ ${VM_DISTRO}_xci_vm:~/bifrost -ssh -F $HOME/.ssh/xci-vm-config ${VM_DISTRO}_xci_vm "cd ~/bifrost && ./bifrost_test.sh" +ssh -F $HOME/.ssh/xci-vm-config ${VM_DISTRO}_xci_vm "cd ~/bifrost/releng-xci && ./bifrost_test.sh" diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 93ca18708..ca14cd9bc 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -195,8 +195,8 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'bifrost-verify-.*' - - 'bifrost-periodic-.*' + - '.*-bifrost-verify-.*' + - '.*-bifrost-periodic-.*' - 'osa-verify-.*' - 'osa-periodic-.*' block-level: 'NODE' |