diff options
Diffstat (limited to 'jjb')
35 files changed, 1015 insertions, 235 deletions
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index b8ae75a1f..94d464259 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -31,7 +31,9 @@ elif [[ "$DEPLOY_SCENARIO" == *gate* ]]; then fi # Dev or RPM/ISO build -if [[ "$ARTIFACT_VERSION" =~ dev ]]; then +# For upstream deployments we currently only use git repo and not RPM +# Need to decide after Fraser if we want to use RPM or not for upstream +if [[ "$ARTIFACT_VERSION" =~ dev || "$DEPLOY_SCENARIO" =~ "upstream" ]]; then # Settings for deploying from git workspace DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy" NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network" @@ -119,7 +121,7 @@ if [[ "$JOB_NAME" =~ "virtual" ]]; then DEPLOY_CMD="${DEPLOY_CMD} -e csit-environment.yaml" fi if [[ "$PROMOTE" == "True" ]]; then - DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 2" + DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 1" fi else # settings for bare metal deployment @@ -134,6 +136,11 @@ else DEPLOY_CMD="${DEPLOY_CMD} -i ${INVENTORY_FILE}" fi +if [[ "$DEPLOY_SCENARIO" =~ "upstream" ]]; then + echo "Upstream deployment detected" + DEPLOY_CMD="${DEPLOY_CMD} --upstream" +fi + if [ "$IPV6_FLAG" == "True" ]; then NETWORK_FILE="${NETWORK_SETTINGS_DIR}/network_settings_v6.yaml" elif echo ${DEPLOY_SCENARIO} | grep fdio; then diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh index 68baf59da..066d61203 100755 --- a/jjb/apex/apex-download-artifact.sh +++ b/jjb/apex/apex-download-artifact.sh @@ -10,7 +10,30 @@ echo [[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY -if [[ "$ARTIFACT_VERSION" =~ dev ]]; then +if [ -z "$DEPLOY_SCENARIO" ]; then + echo "Deploy scenario not set!" + exit 1 +elif [[ "$DEPLOY_SCENARIO" == *gate* ]]; then + echo "Detecting Gating scenario..." + if [ -z "$GERRIT_EVENT_COMMENT_TEXT" ]; then + echo "ERROR: Gate job triggered without comment!" + exit 1 + else + DEPLOY_SCENARIO=$(echo ${GERRIT_EVENT_COMMENT_TEXT} | grep start-gate-scenario | grep -Eo 'os-.*$') + if [ -z "$DEPLOY_SCENARIO" ]; then + echo "ERROR: Unable to detect scenario in Gerrit Comment!" + echo "Format of comment to trigger gate should be 'start-gate-scenario: <scenario>'" + exit 1 + else + echo "Gate scenario detected: ${DEPLOY_SCENARIO}" + fi + fi +fi + +# if upstream we do not need to download anything +if [[ "$DEPLOY_SCENARIO" =~ upstream ]]; then + echo "Upstream deployment detected, skipping download artifact" +elif [[ "$ARTIFACT_VERSION" =~ dev ]]; then # dev build GERRIT_PATCHSET_NUMBER=$(echo $GERRIT_REFSPEC | grep -Eo '[0-9]+$') export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}_${GERRIT_PATCHSET_NUMBER}" diff --git a/jjb/apex/apex-jjb-renderer.py b/jjb/apex/apex-jjb-renderer.py index 6fd83afe9..b67784376 100644 --- a/jjb/apex/apex-jjb-renderer.py +++ b/jjb/apex/apex-jjb-renderer.py @@ -12,30 +12,38 @@ import yaml from jinja2 import Environment from jinja2 import FileSystemLoader -gspathname = dict() -branch = dict() -build_slave = dict() -env = Environment(loader=FileSystemLoader('./'), autoescape=True) -with open('scenarios.yaml.hidden') as _: - scenarios = yaml.safe_load(_) +def render_jjb(): + """Render JJB output from scenarios.yaml.hidden file and jinja + template""" -template = env.get_template('apex.yml.j2') + gspathname = dict() + branch = dict() + build_slave = dict() + env = Environment(loader=FileSystemLoader('./'), autoescape=True) -print("Scenarios are: ") -pprint.pprint(scenarios) + with open('scenarios.yaml.hidden') as _: + scenarios = yaml.safe_load(_) -for stream in scenarios: - if stream == 'master': - gspathname['master'] = '' - branch[stream] = stream - else: - gspathname[stream] = '/' + stream - branch[stream] = 'stable/' + stream - build_slave[stream] = 'apex-baremetal-{}'.format(stream) + template = env.get_template('apex.yml.j2') -output = template.render(scenarios=scenarios, gspathname=gspathname, - branch=branch, build_slave=build_slave) + print("Scenarios are: ") + pprint.pprint(scenarios) -with open('./apex.yml', 'w') as fh: - fh.write(output) + for stream in scenarios: + if stream == 'master': + gspathname['master'] = '' + branch[stream] = stream + else: + gspathname[stream] = '/' + stream + branch[stream] = 'stable/' + stream + build_slave[stream] = 'apex-baremetal-{}'.format(stream) + + output = template.render(scenarios=scenarios, gspathname=gspathname, + branch=branch, build_slave=build_slave) + + with open('./apex.yml', 'w') as fh: + fh.write(output) + +if __name__ == "__main__": + render_jjb() diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index cf29b928c..428835683 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -27,6 +27,7 @@ verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'master' disable_daily: false + disable_promote: false - euphrates: &euphrates branch: 'stable/euphrates' gs-pathname: '/euphrates' @@ -36,6 +37,7 @@ verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'euphrates' disable_daily: true + disable_promote: true - danube: &danube branch: 'stable/danube' gs-pathname: '/danube' @@ -46,6 +48,7 @@ scenario_stream: 'danube' disabled: true disable_daily: true + disable_promote: true scenario: - 'os-nosdn-nofeature-noha': @@ -181,6 +184,10 @@ - 'baremetal' - 'virtual' + os_version: + - 'pike' + - 'queens' + - 'master' - job-template: name: 'apex-runner-cperf-{stream}' @@ -530,7 +537,7 @@ abort-all-job: false git-revision: false - multijob: - name: Dovetail + name: Dovetail-proposed_tests condition: ALWAYS projects: - name: 'dovetail-apex-baremetal-proposed_tests-{scenario_stream}' @@ -539,7 +546,22 @@ predefined-parameters: DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/" + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ + && $BUILD_NUMBER % 2 == 1" + abort-all-job: false + git-revision: false + - multijob: + name: Dovetail-default + condition: ALWAYS + projects: + - name: 'dovetail-apex-baremetal-default-{scenario_stream}' + node-parameters: true + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + kill-phase-on: NEVER + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ + && $BUILD_NUMBER % 2 == 0" abort-all-job: false git-revision: false - multijob: @@ -676,7 +698,7 @@ # branch: branch (eg. stable) node: '{virtual-slave}' project-type: 'multijob' - disabled: false + disabled: '{disable_promote}' scm: - git-scm @@ -703,28 +725,13 @@ builders: - multijob: - name: build - condition: SUCCESSFUL - projects: - - name: 'apex-build-{stream}' - current-parameters: false - predefined-parameters: | - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - - multijob: name: deploy-virtual condition: SUCCESSFUL projects: - name: 'apex-deploy-virtual-{stream}' current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l3-csit-noha + DEPLOY_SCENARIO=os-odl-{os_version}_upstream-noha OPNFV_CLEAN=yes GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC @@ -742,7 +749,7 @@ - name: 'functest-apex-virtual-suite-{stream}' current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l3-nofeature-noha + DEPLOY_SCENARIO=os-odl-{os_version}_upstream-noha FUNCTEST_SUITE_NAME=tempest_smoke_serial GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index ab65c4e2b..0bedd3560 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -27,6 +27,7 @@ verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'master' disable_daily: false + disable_promote: false - euphrates: &euphrates branch: 'stable/euphrates' gs-pathname: '/euphrates' @@ -36,6 +37,7 @@ verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'euphrates' disable_daily: true + disable_promote: true - danube: &danube branch: 'stable/danube' gs-pathname: '/danube' @@ -46,6 +48,7 @@ scenario_stream: 'danube' disabled: true disable_daily: true + disable_promote: true scenario: {%- for stream in scenarios %} @@ -59,6 +62,10 @@ - 'baremetal' - 'virtual' + os_version: + - 'pike' + - 'queens' + - 'master' - job-template: name: 'apex-runner-cperf-{stream}' @@ -408,7 +415,7 @@ abort-all-job: false git-revision: false - multijob: - name: Dovetail + name: Dovetail-proposed_tests condition: ALWAYS projects: - name: 'dovetail-apex-baremetal-proposed_tests-{scenario_stream}' @@ -417,7 +424,22 @@ predefined-parameters: DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/" + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ + && $BUILD_NUMBER % 2 == 1" + abort-all-job: false + git-revision: false + - multijob: + name: Dovetail-default + condition: ALWAYS + projects: + - name: 'dovetail-apex-baremetal-default-{scenario_stream}' + node-parameters: true + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + kill-phase-on: NEVER + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ + && $BUILD_NUMBER % 2 == 0" abort-all-job: false git-revision: false - multijob: @@ -554,7 +576,7 @@ # branch: branch (eg. stable) node: '{virtual-slave}' project-type: 'multijob' - disabled: false + disabled: '{disable_promote}' scm: - git-scm @@ -581,28 +603,13 @@ builders: - multijob: - name: build - condition: SUCCESSFUL - projects: - - name: 'apex-build-{stream}' - current-parameters: false - predefined-parameters: | - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: false - kill-phase-on: FAILURE - abort-all-job: true - git-revision: true - - multijob: name: deploy-virtual condition: SUCCESSFUL projects: - name: 'apex-deploy-virtual-{stream}' current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l3-csit-noha + DEPLOY_SCENARIO=os-odl-{os_version}_upstream-noha OPNFV_CLEAN=yes GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC @@ -620,7 +627,7 @@ - name: 'functest-apex-virtual-suite-{stream}' current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l3-nofeature-noha + DEPLOY_SCENARIO=os-odl-{os_version}_upstream-noha FUNCTEST_SUITE_NAME=tempest_smoke_serial GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC diff --git a/jjb/bottlenecks/bottlenecks-ci-jobs.yml b/jjb/bottlenecks/bottlenecks-ci-jobs.yml index 1d31f2775..072117ae3 100644 --- a/jjb/bottlenecks/bottlenecks-ci-jobs.yml +++ b/jjb/bottlenecks/bottlenecks-ci-jobs.yml @@ -73,6 +73,12 @@ suite: - 'posca_stress_traffic' - 'posca_stress_ping' + - 'posca_factor_multistack_storage' + - 'posca_factor_multistack_storage_parallel' + - 'posca_feature_moon_resources' + - 'posca_feature_moon_tenants' + - 'posca_feature_vnf_scale_out' + - 'posca_factor_soak_throughputs' jobs: - 'bottlenecks-{installer}-{suite}-{pod}-daily-{stream}' diff --git a/jjb/bottlenecks/bottlenecks-run-suite.sh b/jjb/bottlenecks/bottlenecks-run-suite.sh index ebd905e98..7a3db00c4 100644 --- a/jjb/bottlenecks/bottlenecks-run-suite.sh +++ b/jjb/bottlenecks/bottlenecks-run-suite.sh @@ -136,13 +136,12 @@ if [[ $SUITE_NAME == *posca* ]]; then # Running test cases through Bottlenecks docker if [[ $SUITE_NAME == posca_stress_traffic ]]; then TEST_CASE=posca_factor_system_bandwidth - testcase_cmd="docker exec bottlenecks-load-master python ${POSCA_SCRIPT}/../run_testsuite.py testcase $TEST_CASE $REPORT" - echo "BOTTLENECKS INFO: running test case ${TEST_CASE} with report indicator: ${testcase_cmd}" - ${testcase_cmd} >$redirect elif [[ $SUITE_NAME == posca_stress_ping ]]; then TEST_CASE=posca_factor_ping - testcase_cmd="docker exec bottlenecks-load-master python ${POSCA_SCRIPT}/../run_testsuite.py testcase $TEST_CASE $REPORT" - echo "BOTTLENECKS INFO: running test case ${TEST_CASE} with report indicator: ${testcase_cmd}" - ${testcase_cmd} >$redirect + else + TEST_CASE=$SUITE_NAME fi + testcase_cmd="docker exec bottlenecks-load-master python ${POSCA_SCRIPT}/../run_testsuite.py testcase $TEST_CASE $REPORT" + echo "BOTTLENECKS INFO: running test case ${TEST_CASE} with report indicator: ${testcase_cmd}" + ${testcase_cmd} >$redirect fi diff --git a/jjb/clover/clover-project.yml b/jjb/clover/clover-project.yml new file mode 100644 index 000000000..b31f3deb2 --- /dev/null +++ b/jjb/clover/clover-project.yml @@ -0,0 +1,166 @@ +--- +################################################### +# All the jobs except verify have been removed! +# They will only be enabled on request by projects! +################################################### +- project: + name: clover + + project: '{name}' + + jobs: + - 'clover-verify-{stream}' + - 'clover-daily-upload-{stream}' + - 'clover-daily-deploy-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + +- job-template: + name: 'clover-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + # yamllint disable rule:line-length + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + # yamllint enable rule:line-length + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - shell: | + cd $WORKSPACE/ci + ./verify.sh + +- job-template: + name: 'clover-daily-upload-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + scm: + - git-scm + + triggers: + - timed: '0 11 * * *' + + wrappers: + - fix-workspace-permissions + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + - 'clover-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - shell: | + cd $WORKSPACE/ci + ./upload.sh + +- job-template: + name: 'clover-daily-deploy-{stream}' + + project-type: freestyle + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'clover-daily-deploy-.*?' + - 'container4nfv-daily-deploy-.*?' + block-level: 'NODE' + + wrappers: + - timeout: + timeout: 180 + abort: true + - fix-workspace-permissions + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + # yamllint disable rule:line-length + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + # yamllint enable rule:line-length + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - huawei-virtual6 + default-slaves: + - huawei-virtual6 + + scm: + - git-scm + + triggers: + - timed: '0 12 * * *' + + builders: + - shell: | + cd $WORKSPACE/ci + ./deploy.sh + +################### +# parameter macros +################### +- parameter: + name: 'clover-defaults' + parameters: + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 357f24e37..44cb84d8e 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -289,6 +289,26 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + - project: 'bottlenecks-compass-posca_factor_multistack_storage_parallel-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - project: 'bottlenecks-compass-posca_factor_soak_throughputs-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' - conditional-step: condition-kind: and condition-operands: diff --git a/jjb/container4nfv/container4nfv-project.yml b/jjb/container4nfv/container4nfv-project.yml index 03bbb655e..1ab50ae30 100644 --- a/jjb/container4nfv/container4nfv-project.yml +++ b/jjb/container4nfv/container4nfv-project.yml @@ -109,8 +109,21 @@ properties: - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'clover-daily-deploy-.*?' + - 'container4nfv-daily-deploy-.*?' + block-level: 'NODE' wrappers: + - timeout: + timeout: 240 + abort: true - fix-workspace-permissions parameters: diff --git a/jjb/container4nfv/yardstick-arm64.yml b/jjb/container4nfv/yardstick-arm64.yml index bd1d8aa85..9b8b271df 100644 --- a/jjb/container4nfv/yardstick-arm64.yml +++ b/jjb/container4nfv/yardstick-arm64.yml @@ -5,7 +5,7 @@ - project: name: yardstick-arm64 - project: '{name}' + project: 'yardstick' # ------------------------------- # BRANCH ANCHORS @@ -84,6 +84,10 @@ name: CI_DEBUG default: 'false' description: "Show debut output information" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration." scm: - git-scm diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index a9a0da2ee..cc2dc5a2f 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -190,8 +190,10 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'daisy.*-deploy-({pod})?-daily-.*' - 'daisy-kolla-build-.*' + - '{installer}-(build|deploy|test)-daily-(euphrates|master)' + - '{installer}-deploy-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' block-level: 'NODE' parameters: @@ -333,7 +335,7 @@ - trigger: name: 'daisy-os-nosdn-ovs_dpdk-noha-zte-pod9-daily-master-trigger' triggers: - - timed: '0 10 * * *' + - timed: '0 10,20 * * *' # Basic HA Scenarios - trigger: name: 'daisy-os-nosdn-nofeature-ha-zte-pod9-daily-master-trigger' diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index 006153f6f..c4edd7738 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -215,6 +215,13 @@ enabled: true max-total: 6 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + block-level: 'NODE' scm: - git-scm diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index ba76b8abb..8c04ffd53 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -19,7 +19,7 @@ installer: - 'apex' - 'fuel' - # - 'joid' + - 'daisy' arch: - 'x86_64' @@ -29,19 +29,17 @@ - 'sample' - 'congress' - task: - - verify: - auto-trigger-name: 'doctor-verify' - exclude: - installer: 'apex' arch: 'aarch64' + - installer: 'daisy' + arch: 'aarch64' jobs: - 'doctor-verify-{stream}' - 'doctor-verify-unit-test-{stream}' - - 'doctor-{task}-{inspector}-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + - 'doctor-verify-{inspector}-{stream}' + - 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' - job-template: name: 'doctor-verify-{stream}' @@ -76,6 +74,7 @@ file-paths: - compare-type: ANT pattern: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -117,16 +116,17 @@ - 'doctor-verify-unit-test-publishers-macro' - job-template: - name: 'doctor-{task}-{inspector}-{stream}' + name: 'doctor-verify-{inspector}-{stream}' disabled: '{obj:disabled}' project-type: 'multijob' scm: - git-scm-gerrit triggers: - - '{auto-trigger-name}': + - 'doctor-verify': project: '{project}' branch: '{branch}' files: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -137,7 +137,7 @@ name: 'doctor-verify-apex-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-apex-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-apex-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -151,7 +151,7 @@ name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -165,7 +165,20 @@ name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-aarch64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-aarch64-{stream}' + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + kill-phase-on: FAILURE + current-parameters: true + git-revision: true + - multijob: + name: 'doctor-verify-daisy-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-verify-daisy-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -177,7 +190,7 @@ git-revision: true - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + name: 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' disabled: '{obj:disabled}' node: 'doctor-{installer}-{arch}' wrappers: diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index ec879e325..d01128385 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -24,6 +24,9 @@ mkdir -p ${DOVETAIL_HOME} DOVETAIL_CONFIG=${DOVETAIL_HOME}/pre_config mkdir -p ${DOVETAIL_CONFIG} +DOVETAIL_IMAGES=${DOVETAIL_HOME}/images +mkdir -p ${DOVETAIL_IMAGES} + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" sshkey="" @@ -90,13 +93,24 @@ fi cat $OPENRC if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then + compass_repo=${WORKSPACE}/compass4nfv/ + git clone https://github.com/opnfv/compass4nfv.git ${compass_repo} >/dev/null + sudo pip install shyaml + scenario_file=${compass_repo}/deploy/conf/hardware_environment/$NODE_NAME/os-nosdn-nofeature-ha.yml + ipmiIp=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiIp) + ipmiPass=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiPass) + ipmiUser=root + jumpserver_ip=$(ifconfig | grep -A 5 docker0 | grep "inet addr" | cut -d ':' -f 2 | cut -d ' ' -f 1) + cat << EOF >${DOVETAIL_CONFIG}/pod.yaml nodes: -- {ip: 10.1.0.52, name: node1, password: root, role: controller, user: root} +- {ip: ${jumpserver_ip}, name: node0, password: root, role: Jumpserver, user: root} +- {ip: 10.1.0.50, name: node1, password: root, role: controller, user: root, + ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}} - {ip: 10.1.0.51, name: node2, password: root, role: controller, user: root} -- {ip: 10.1.0.50, name: node3, password: root, role: controller, user: root} -- {ip: 10.1.0.54, name: node4, password: root, role: compute, user: root} -- {ip: 10.1.0.53, name: node5, password: root, role: compute, user: root} +- {ip: 10.1.0.52, name: node3, password: root, role: controller, user: root} +- {ip: 10.1.0.53, name: node4, password: root, role: compute, user: root} +- {ip: 10.1.0.54, name: node5, password: root, role: compute, user: root} EOF fi @@ -151,6 +165,14 @@ if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then fi if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then + sudo chmod 666 ${DOVETAIL_CONFIG}/pod.yaml + echo "Adapt process info for $INSTALLER_TYPE ..." + attack_process='rabbitmq' + cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml +process_info: +- {testcase_name: dovetail.ha.tc010, attack_process: ${attack_process}} + +EOF echo "file ${DOVETAIL_CONFIG}/pod.yaml:" cat ${DOVETAIL_CONFIG}/pod.yaml else @@ -189,7 +211,7 @@ if [[ ! -f ${ubuntu_image} ]]; then echo "Download image ubuntu-16.04-server-cloudimg-amd64-disk1.img ..." wget -q -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${image_path} fi -sudo cp ${ubuntu_image} ${DOVETAIL_CONFIG} +sudo cp ${ubuntu_image} ${DOVETAIL_IMAGES} # functest needs to download this image first before running cirros_image=${image_path}/cirros-0.3.5-x86_64-disk.img @@ -197,7 +219,7 @@ if [[ ! -f ${cirros_image} ]]; then echo "Download image cirros-0.3.5-x86_64-disk.img ..." wget -q -nc http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img -P ${image_path} fi -sudo cp ${cirros_image} ${DOVETAIL_CONFIG} +sudo cp ${cirros_image} ${DOVETAIL_IMAGES} # snaps_smoke test case needs to download this image first before running ubuntu14_image=${image_path}/ubuntu-14.04-server-cloudimg-amd64-disk1.img @@ -205,7 +227,7 @@ if [[ ! -f ${ubuntu14_image} ]]; then echo "Download image ubuntu-14.04-server-cloudimg-amd64-disk1.img ..." wget -q -nc https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img -P ${image_path} fi -sudo cp ${ubuntu14_image} ${DOVETAIL_CONFIG} +sudo cp ${ubuntu14_image} ${DOVETAIL_IMAGES} # cloudify_ims test case needs to download these 2 images first before running cloudify_image=${image_path}/cloudify-manager-premium-4.0.1.qcow2 @@ -213,13 +235,13 @@ if [[ ! -f ${cloudify_image} ]]; then echo "Download image cloudify-manager-premium-4.0.1.qcow2 ..." wget -q -nc http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 -P ${image_path} fi -sudo cp ${cloudify_image} ${DOVETAIL_CONFIG} +sudo cp ${cloudify_image} ${DOVETAIL_IMAGES} trusty_image=${image_path}/trusty-server-cloudimg-amd64-disk1.img if [[ ! -f ${trusty_image} ]]; then echo "Download image trusty-server-cloudimg-amd64-disk1.img ..." wget -q -nc http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img -P ${image_path} fi -sudo cp ${trusty_image} ${DOVETAIL_CONFIG} +sudo cp ${trusty_image} ${DOVETAIL_IMAGES} opts="--privileged=true -id" @@ -236,10 +258,8 @@ fi echo "Dovetail: Pulling image ${DOCKER_REPO}:${DOCKER_TAG}" docker pull ${DOCKER_REPO}:$DOCKER_TAG >$redirect -env4bgpvpn="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP}" - cmd="docker run ${opts} -e DOVETAIL_HOME=${DOVETAIL_HOME} ${docker_volume} ${dovetail_home_volume} \ - ${sshkey} ${env4bgpvpn} ${DOCKER_REPO}:${DOCKER_TAG} /bin/bash" + ${sshkey} ${DOCKER_REPO}:${DOCKER_TAG} /bin/bash" echo "Dovetail: running docker run command: ${cmd}" ${cmd} >${redirect} sleep 5 @@ -299,8 +319,9 @@ sudo cp -r ${DOVETAIL_HOME}/results ./ # PRIMARY_GROUP=$(id -gn $CURRENT_USER) # sudo chown -R ${CURRENT_USER}:${PRIMARY_GROUP} ${WORKSPACE}/results -#remove useless workspace from yardstick to save disk space +#remove useless files to save disk space sudo rm -rf ./results/workspace +sudo rm -f ./results/yardstick.img echo "Dovetail: done!" diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index 432bbbb8b..dfadf9a37 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -161,28 +161,35 @@ if [ "${INSTALLER_TYPE}" == 'fuel' ]; then envs="${envs} -e POD_ARCH=${COMPUTE_ARCH}" fi -volumes="${images_vol} ${results_vol} ${sshkey_vol} ${rc_file_vol} ${cacert_file_vol}" -set +e +if [[ ${INSTALLER_TYPE} == 'compass' && ${DEPLOY_SCENARIO} =~ 'sfc' ]]; then + ssh_key="/tmp/id_rsa" + user_config="/tmp/openstack_user_config.yml" + docker cp compass-tasks:/root/.ssh/id_rsa $ssh_key + docker cp compass-tasks:/etc/openstack_deploy/openstack_user_config.yml $user_config + sshkey_vol="-v ${ssh_key}:/root/.ssh/id_rsa" + userconfig_vol="-v ${user_config}:${user_config}" + envs="${envs} -e EXTERNAL_NETWORK=${EXTERNAL_NETWORK}" +fi + + +volumes="${images_vol} ${results_vol} ${sshkey_vol} ${userconfig_vol} ${rc_file_vol} ${cacert_file_vol}" +set +e -if [[ ${DEPLOY_SCENARIO} =~ ^os-.* ]]; then - [[ ${BRANCH##*/} == "master" ]] && check_os_deployment - if [ ${FUNCTEST_MODE} == 'testcase' ]; then - echo "FUNCTEST_MODE=testcase, FUNCTEST_SUITE_NAME=${FUNCTEST_SUITE_NAME}" - run_test ${FUNCTEST_SUITE_NAME} - elif [ ${FUNCTEST_MODE} == 'tier' ]; then - echo "FUNCTEST_MODE=tier, FUNCTEST_TIER=${FUNCTEST_TIER}" - tiers=(${FUNCTEST_TIER}) - run_tiers ${tiers} +[[ ${BRANCH##*/} == "master" ]] && check_os_deployment +if [ ${FUNCTEST_MODE} == 'testcase' ]; then + echo "FUNCTEST_MODE=testcase, FUNCTEST_SUITE_NAME=${FUNCTEST_SUITE_NAME}" + run_test ${FUNCTEST_SUITE_NAME} +elif [ ${FUNCTEST_MODE} == 'tier' ]; then + echo "FUNCTEST_MODE=tier, FUNCTEST_TIER=${FUNCTEST_TIER}" + tiers=(${FUNCTEST_TIER}) + run_tiers ${tiers} +else + if [ ${DEPLOY_TYPE} == 'baremetal' ]; then + tiers=(healthcheck smoke features vnf parser) else - if [ ${DEPLOY_TYPE} == 'baremetal' ]; then - tiers=(healthcheck smoke features vnf parser) - else - tiers=(healthcheck smoke features) - fi - run_tiers ${tiers} + tiers=(healthcheck smoke features) fi -else - echo "k8 deployment has not been supported by functest yet" + run_tiers ${tiers} fi diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index a58e0bf3f..8ffbe9e63 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -199,7 +199,7 @@ - 'daily': job-timeout: 360 - 'arm-daily': - job-timeout: 360 + job-timeout: 480 jobs: - 'functest-{installer}-{pod}-{testsuite}-{stream}' diff --git a/jjb/functest/functest-k8.sh b/jjb/functest/functest-k8.sh index 3f08e4a0b..9017354cc 100755 --- a/jjb/functest/functest-k8.sh +++ b/jjb/functest/functest-k8.sh @@ -12,14 +12,13 @@ sudo rm -rf $rc_file if [[ ${INSTALLER_TYPE} == 'compass' ]]; then admin_conf_file_vol="-v ${HOME}/admin.conf:/root/.kube/config" - echo "export KUBECONFIG=/root/.kube/config" >> $rc_file echo "export KUBERNETES_PROVIDER=local" >> $rc_file KUBE_MASTER_URL=$(cat ${HOME}/admin.conf|grep server| awk '{print $2}') echo "export KUBE_MASTER_URL=$KUBE_MASTER_URL" >> $rc_file KUBE_MASTER_IP=$(echo $KUBE_MASTER_URL|awk -F'https://|:[0-9]+' '$0=$2') echo "export KUBE_MASTER_IP=$KUBE_MASTER_IP" >> $rc_file elif [[ ${INSTALLER_TYPE} == 'joid' && ${BRANCH} == 'master' ]]; then - admin_conf_file_vol="-v ${HOME}/joid_config/config:/root/joid_config/config" + admin_conf_file_vol="-v ${HOME}/joid_config/config:/root/.kube/config" rc_file=${HOME}/joid_config/k8config else echo "Not supported by other installers yet" @@ -35,25 +34,25 @@ results_vol="-v ${dir_result}:${FUNCTEST_DIR}/results" volumes="${rc_file_vol} ${results_vol} ${admin_conf_file_vol}" -# Set iptables rule to allow forwarding return traffic for container -if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then - sudo iptables -I FORWARD -j RETURN -fi - envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} \ -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ -e BUILD_TAG=${BUILD_TAG} -e DEPLOY_TYPE=${DEPLOY_TYPE}" DOCKER_TAG=`[[ ${BRANCH##*/} == "master" ]] && echo "latest" || echo ${BRANCH##*/}` +set +e + FUNCTEST_IMAGE=opnfv/functest-kubernetes:${DOCKER_TAG} echo "Pulling Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null cmd_opt="run_tests -r -t all" -cmd="docker run --rm --privileged=true ${envs} ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" +cmd="docker run --rm ${envs} ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest k8s test cases, CMD: ${cmd}" eval ${cmd} ret_value=$? + +ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value" +echo 0 > ${ret_val_file} if [ ${ret_value} != 0 ]; then echo ${ret_value} > ${ret_val_file} fi diff --git a/jjb/global-jjb b/jjb/global-jjb new file mode 160000 +Subproject 779110b5cd63f3eabb63598a1be79d9b9ba8546 diff --git a/jjb/global/basic-jobs.yml b/jjb/global/basic-jobs.yml index e55f068ca..a8b9cffbc 100644 --- a/jjb/global/basic-jobs.yml +++ b/jjb/global/basic-jobs.yml @@ -13,7 +13,7 @@ branch: '{stream}' gs-pathname: '' disabled: false - - euphrates: + - fraser: branch: 'stable/{stream}' gs-pathname: '/{stream}' disabled: false diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 86b369bcd..6171a6abd 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -983,6 +983,7 @@ - 'doctor-fuel-x86_64' - 'doctor-fuel-aarch64' - 'doctor-joid-x86_64' + - 'doctor-daisy-x86_64' - 'not-specified' default-slaves: - 'not-specified' diff --git a/jjb/opnfvdocs/project.cfg b/jjb/opnfvdocs/project.cfg index 0722b4036..598269737 100644 --- a/jjb/opnfvdocs/project.cfg +++ b/jjb/opnfvdocs/project.cfg @@ -2,6 +2,7 @@ apex armband availability bottlenecks +clover compass4nfv copper conductor diff --git a/jjb/qtip/qtip-validate-jobs.yml b/jjb/qtip/qtip-validate-jobs.yml index 5c0bb881b..c5bf58391 100644 --- a/jjb/qtip/qtip-validate-jobs.yml +++ b/jjb/qtip/qtip-validate-jobs.yml @@ -34,12 +34,18 @@ pod: zte-virtual5 sut: '' <<: *master + - network: + installer: apex + pod: zte-virtual5 + sut: '' + <<: *master - compute: installer: fuel pod: zte-virtual6 sut: vnf <<: *master + # ------------------------------- # JOB LIST # ------------------------------- diff --git a/jjb/releng/releng-release-create-branch.sh b/jjb/releng/releng-release-create-branch.sh new file mode 100644 index 000000000..83755608e --- /dev/null +++ b/jjb/releng/releng-release-create-branch.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -xe + +# Configure the git user/email as we'll be pushing up changes +git config --global user.name "jenkins-ci" +git config --global user.email "jenkins-opnfv-ci@opnfv.org" + +# Ensure we are able to generate Commit-IDs for new patchsets +curl -kLo .git/hooks/commit-msg https://gerrit.opnfv.org/gerrit/tools/hooks/commit-msg +chmod +x .git/hooks/commit-msg + +# Activate virtualenv, supressing shellcheck warning +# shellcheck source=/dev/null +. $WORKSPACE/venv/bin/activate +pip install -r releases/scripts/requirements.txt + +STREAM=${STREAM:-'nostream'} +RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM") + +for release_file in $RELEASE_FILES; do + python releases/scripts/create_branch.py -f $release_file + python releases/scripts/create_jobs.py -f $release_file + NEW_FILES=$(git status --porcelain --untracked=no | cut -c4-) + if [ -n "$NEW_FILES" ]; then + git add $NEW_FILES + git commit -m "Create Stable Branch Jobs for $(basename $release_file .yaml)" + git push origin HEAD:refs/for/master + fi +done diff --git a/jjb/releng/releng-release-jobs.yml b/jjb/releng/releng-release-jobs.yml new file mode 100644 index 000000000..e31320b81 --- /dev/null +++ b/jjb/releng/releng-release-jobs.yml @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- project: + name: releng-release-jobs + + stream: + - fraser + + jobs: + - 'releng-release-{stream}-verify' + - 'releng-release-{stream}-merge' + + project: 'releng' + +- job-template: + name: 'releng-release-{stream}-verify' + + parameters: + - stream-parameter: + stream: '{stream}' + - project-parameter: + project: '{project}' + branch: 'master' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: 'releng' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' + file-paths: + - compare-type: ANT + pattern: 'releases/{stream}/**' + - compare-type: ANT + pattern: 'releases/schema.yaml' + - compare-type: ANT + pattern: 'releases/scripts/verify_schema.py' + + builders: + - create-virtualenv + - shell: + !include-raw-escape: releng-release-verify.sh + + publishers: + - email-jenkins-admins-on-failure + +- job-template: + name: 'releng-release-{stream}-merge' + + parameters: + - node: + name: SLAVE_NAME + description: 'Only run merge job on build1' + default-slaves: + - lf-build1 + allowed-multiselect: false + ignore-offline-nodes: true + - stream-parameter: + stream: '{stream}' + - project-parameter: + project: '{project}' + branch: 'master' + + scm: + - git-scm-gerrit + + triggers: + - gerrit-trigger-change-merged: + project: '{project}' + branch: 'master' + files: 'releases/{stream}/*' + + builders: + - create-virtualenv + - shell: + !include-raw-escape: releng-release-create-branch.sh + + publishers: + - email-jenkins-admins-on-failure + +- parameter: + name: stream-parameter + parameters: + - string: + name: STREAM + default: '{stream}' + description: "OPNFV Stable Stream" + +- builder: + name: create-virtualenv + builders: + - shell: | + #!/bin/bash + sudo pip install virtualenv + virtualenv $WORKSPACE/venv + . $WORKSPACE/venv/bin/activate + pip install --upgrade pip diff --git a/jjb/releng/releng-release-verify.sh b/jjb/releng/releng-release-verify.sh new file mode 100644 index 000000000..c1262e2c9 --- /dev/null +++ b/jjb/releng/releng-release-verify.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -xe + +# Activate virtualenv, supressing shellcheck warning +# shellcheck source=/dev/null +. $WORKSPACE/venv/bin/activate +pip install -r releases/scripts/requirements.txt + +STREAM=${STREAM:-'nostream'} +RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM") + +# TODO: The create_branch.py should be refactored so it can be used here +# to verify the commit exists that is being added, along with +# jjb/<project> +for release_file in $RELEASE_FILES; do + python releases/scripts/verify_schema.py \ + -s releases/schema.yaml \ + -y $release_file +done diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh index 9219d0020..9d524bcb0 100755 --- a/jjb/xci/bifrost-verify.sh +++ b/jjb/xci/bifrost-verify.sh @@ -17,7 +17,7 @@ cd $WORKSPACE git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD # combine opnfv and upstream scripts/playbooks -/bin/cp -rf $WORKSPACE/releng-xci/bifrost/* $WORKSPACE/ +/bin/cp -rf $WORKSPACE/releng-xci/xci/infra/bifrost/* $WORKSPACE/ cd $WORKSPACE/releng-xci cat > bifrost_test.sh<<EOF diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml index 92eea190f..e4e5705a7 100644 --- a/jjb/xci/xci-merge-jobs.yml +++ b/jjb/xci/xci-merge-jobs.yml @@ -2,51 +2,49 @@ - project: name: 'opnfv-xci-merge' + project: releng-xci # ------------------------------- # branches # ------------------------------- stream: - master: branch: '{stream}' + disabled: false # ------------------------------- # distros # ------------------------------- distro: - ubuntu: disabled: false + kill-phase-on: FAILURE + abort-all-job: true - centos: - disabled: false + disabled: true + kill-phase-on: NEVER + abort-all-job: false - opensuse: disabled: false + kill-phase-on: FAILURE + abort-all-job: true # ------------------------------- - # type - # ------------------------------- - type: - - virtual - # ------------------------------- - # patch verification phases + # postmerge promotion phases # ------------------------------- phase: - 'deploy' - 'healthcheck' - - 'promote' - # ------------------------------- - # scenarios - # ------------------------------- - scenario: - - 'os-odl-sfc': - project: sfc # ------------------------------- # jobs # ------------------------------- jobs: - - 'xci-{scenario}-{type}-{distro}-merge-{stream}' - - 'xci-{phase}-{type}-{distro}-merge-{stream}' + - 'xci-merge-virtual-{stream}' + - 'xci-merge-promote-virtual-{stream}' + - 'xci-merge-{distro}-virtual-{stream}' + - 'xci-merge-{distro}-{phase}-virtual-{stream}' # ------------------------------- # job templates # ------------------------------- - job-template: - name: 'xci-{scenario}-{type}-{distro}-merge-{stream}' + name: 'xci-merge-virtual-{stream}' project-type: multijob @@ -56,22 +54,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'xci-verify-{distro}-.*' - - 'xci-.*-{distro}-merge-.*' - - 'openstack-bifrost-verify-{distro}-.*' - - 'xci-osa-verify-{distro}-.*' - - 'xci-osa-periodic-{distro}-.*' - - 'xci-(os|k8s).*?-virtual-{distro}-.*' - block-level: 'NODE' - - throttle: - max-per-node: 2 - max-total: 10 - categories: - - xci-verify-virtual - option: category wrappers: - ssh-agent-wrapper @@ -80,12 +62,7 @@ - fix-workspace-permissions scm: - - git: - credentials-id: '$SSH_CREDENTIAL_ID' - url: 'https://gerrit.opnfv.org/gerrit/releng-xci' - branches: - - 'origin/master' - timeout: 15 + - git-scm-gerrit triggers: - gerrit: @@ -96,7 +73,24 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'ANT' - project-pattern: '{obj:project}' + project-pattern: 'releng-xci' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'false' + forbidden-file-paths: + - compare-type: ANT + pattern: 'xci/scripts/vm/**' + - compare-type: ANT + pattern: 'docs/**' + - compare-type: ANT + pattern: 'prototypes/**' + - compare-type: ANT + pattern: 'upstream/**' + - compare-type: ANT + pattern: 'INFO.yaml' + - project-compare-type: 'REG_EXP' + project-pattern: 'sfc|sdnvpn' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -109,14 +103,165 @@ parameters: - project-parameter: - project: '{obj:project}' + project: $GERRIT_PROJECT + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'xci-virtual' + - string: + name: CI_LOOP + default: 'merge' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' + - string: + name: DISTRO + default: 'all' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + builders: + - 'xci-merge-set-scenario-macro' + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - description-setter: + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" + - multijob: + name: deploy and test + condition: SUCCESSFUL + projects: + - name: 'xci-merge-opensuse-virtual-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=opensuse + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - name: 'xci-merge-ubuntu-virtual-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=ubuntu + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - name: 'xci-merge-centos-virtual-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=centos + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + node-parameters: false + kill-phase-on: NEVER + abort-all-job: false + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - multijob: + name: promote + condition: SUCCESSFUL + projects: + - name: 'xci-merge-promote-virtual-{stream}' + current-parameters: true + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + +- job-template: + name: 'xci-merge-{distro}-virtual-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-verify-{distro}-.*' + - 'xci-merge-{distro}-.*' + - 'openstack-bifrost-verify-{distro}-.*' + - 'xci-osa-verify-{distro}-.*' + - 'xci-osa-periodic-{distro}-.*' + block-level: 'NODE' + - throttle: + max-per-node: 1 + max-total: 3 + categories: + - xci-verify-virtual + option: category + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + scm: + - git-scm-gerrit + + parameters: + - project-parameter: + project: $GERRIT_PROJECT branch: '{branch}' - label: name: SLAVE_LABEL default: 'xci-virtual' - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + name: CI_LOOP + default: 'merge' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' + - string: + name: DISTRO + default: '{distro}' - string: name: CLEAN_DIB_IMAGES default: 'true' @@ -126,46 +271,48 @@ description: 'Git URL to use on this Jenkins Slave' builders: + - 'xci-merge-set-scenario-macro' + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - multijob: name: deploy condition: SUCCESSFUL projects: - - name: 'xci-deploy-{type}-{distro}-merge-{stream}' + - name: 'xci-merge-{distro}-deploy-virtual-{stream}' current-parameters: true predefined-parameters: | - PROJECT=$PROJECT DISTRO={distro} - DEPLOY_SCENARIO={scenario} CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true + kill-phase-on: '{kill-phase-on}' + abort-all-job: '{abort-all-job}' - multijob: name: healthcheck condition: SUCCESSFUL projects: - - name: 'xci-healthcheck-{type}-{distro}-merge-{stream}' + - name: 'xci-merge-{distro}-healthcheck-virtual-{stream}' current-parameters: true predefined-parameters: | - PROJECT=$PROJECT DISTRO={distro} - DEPLOY_SCENARIO={scenario} CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true - kill-phase-on: NEVER - abort-all-job: true + kill-phase-on: '{kill-phase-on}' + abort-all-job: '{abort-all-job}' publishers: - postbuildscript: @@ -176,11 +323,11 @@ !include-raw: ./xci-cleanup.sh - job-template: - name: 'xci-{phase}-{type}-{distro}-merge-{stream}' + name: 'xci-merge-{distro}-{phase}-virtual-{stream}' disabled: false - concurrent: true + concurrent: false properties: - logrotate-default @@ -196,13 +343,13 @@ parameters: - string: name: PROJECT - default: 'releng-xci' + default: $GERRIT_PROJECT - string: name: DISTRO default: 'ubuntu' - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-noha' + name: CI_LOOP + default: 'merge' - string: name: FUNCTEST_MODE default: 'tier' @@ -219,40 +366,94 @@ name: OPNFV_RELENG_DEV_PATH default: $WORKSPACE/ - string: - name: INSTALLER_TYPE - default: 'osa' - - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - ssh-agent-wrapper - build-timeout: timeout: 240 - fix-workspace-permissions scm: - - git: - credentials-id: '$SSH_CREDENTIAL_ID' - url: 'https://gerrit.opnfv.org/gerrit/releng-xci' - branches: - - 'origin/master' - timeout: 15 + - git-scm-gerrit builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - 'xci-merge-{phase}-macro' +- job-template: + name: 'xci-merge-promote-virtual-{stream}' + + disabled: false + + concurrent: false + + properties: + - logrotate-default + + parameters: + - string: + name: PROJECT + default: $GERRIT_PROJECT + - string: + name: CI_LOOP + default: 'merge' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' + - string: + name: DISTRO + default: 'all' + - string: + name: OPNFV_RELENG_DEV_PATH + default: $WORKSPACE/ + - string: + name: LOCAL_PROMOTION_METADATA_FILE + default: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - string: + name: REMOTE_PROMOTION_METADATA_FILE + default: "gs://artifacts.opnfv.org/xci/pipeline/merge/$DEPLOY_SCENARIO.properties" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + builders: + - description-setter: + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" + - 'xci-merge-promote-macro' + # ------------------------------- # builder macros # ------------------------------- - builder: + name: 'xci-merge-set-scenario-macro' + builders: + - shell: + !include-raw: ./xci-set-scenario.sh + +- builder: name: 'xci-merge-deploy-macro' builders: - shell: !include-raw: ./xci-start-new-vm.sh + - shell: !include-raw: ./xci-start-deployment.sh diff --git a/jjb/xci/xci-promote.sh b/jjb/xci/xci-promote.sh index 62818abee..d8d61bee2 100755 --- a/jjb/xci/xci-promote.sh +++ b/jjb/xci/xci-promote.sh @@ -23,11 +23,29 @@ if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then exit 0 fi -# skip the deployment if the scenario is not supported on this distro -OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml -if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then - echo "# SKIPPED: Scenario $DEPLOY_SCENARIO is NOT supported on $DISTRO" - exit 0 +# fail if promotion metadata file doesn't exist +if [ ! -f $LOCAL_PROMOTION_METADATA_FILE ]; then + echo "Unable to find promotion metadata file $LOCAL_PROMOTION_METADATA_FILE" + echo "Skipping promotion!" + exit 1 fi -echo "Hello World!" +# put additional info into the metadata file so we can use that for displaying the information +echo "PROMOTED_BY=$BUILD_URL" >> $LOCAL_PROMOTION_METADATA_FILE +echo "PROMOTED_ON=$(date -u '+%F_%H:%M'UTC)" >> $LOCAL_PROMOTION_METADATA_FILE + +# upload promotion metadata file to OPNFV artifact repo +echo "Storing promotion metadata as $REMOTE_PROMOTION_METADATA_FILE" +gsutil cp $LOCAL_PROMOTION_METADATA_FILE $REMOTE_PROMOTION_METADATA_FILE > /dev/null 2>&1 + +# update the file metadata on gs to prevent the use of cached version of the file +gsutil -m setmeta -r -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + $REMOTE_PROMOTION_METADATA_FILE > /dev/null 2>&1 + +# log the metadata to console +echo "Stored the metadata for $DEPLOY_SCENARIO" +echo "---------------------------------------------------------------------------------" +gsutil cat $REMOTE_PROMOTION_METADATA_FILE +echo "---------------------------------------------------------------------------------" +echo "Scenario $DEPLOY_SCENARIO has successfully been promoted!" diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh index 60b48cf7f..aa98e07aa 100755 --- a/jjb/xci/xci-run-functest.sh +++ b/jjb/xci/xci-run-functest.sh @@ -7,9 +7,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -set -o errexit set -o nounset -set -o pipefail #---------------------------------------------------------------------- # This script is used by CI and executed by Jenkins jobs. @@ -51,10 +49,18 @@ if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENT exit 0 fi -ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci/xci && PATH=/home/devuser/.local/bin:$PATH ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/prepare-functest.yml" +# set XCI_VENV for ansible +export XCI_VENV=/home/devuser/releng-xci/venv + +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "source $XCI_VENV/bin/activate; cd releng-xci/xci && ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/prepare-functest.yml" echo "Running functest" ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "/root/run-functest.sh" echo "Functest log" echo "---------------------------------------------------------------------------------" ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "cat /root/results/functest.log" echo "---------------------------------------------------------------------------------" +# check the log to see if we have any error +if ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "grep -q 'FAIL' /root/results/functest.log"; then + echo "Error: Functest failed!" + exit 1 +fi diff --git a/jjb/xci/xci-set-scenario.sh b/jjb/xci/xci-set-scenario.sh index 62b26c019..c3012cd88 100755 --- a/jjb/xci/xci-set-scenario.sh +++ b/jjb/xci/xci-set-scenario.sh @@ -42,20 +42,22 @@ set -o pipefail function override_generic_scenario() { echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + # ensure the metadata we record is consistent for all types of patches including skipped ones + # extract releng-xci sha + XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha which is same as releng-xci sha for generic scenarios + SCENARIO_SHA=$XCI_SHA + # process topic branch names - if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then - # skip the real verification - echo "Skipping verify!" - echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties - echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties - exit 0 - elif [[ "$GERRIT_TOPIC" =~ 'force-verify' ]]; then - # Run the deployment with default installer and scenario when multiple things change - # and we want to force that. - echo "Recording the installer 'osa' and scenario 'os-nosdn-nofeature' for downstream jobs" - echo "Forcing CI verification of default scenario and installer!" + if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment|force-verify ]]; then + [[ "$GERRIT_TOPIC" =~ force-verify ]] && echo "Forcing CI verification using default scenario and installer!" + [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]] && echo "Skipping verification!" echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + echo "XCI_SHA=$XCI_SHA" >> $WORK_DIRECTORY/scenario.properties + echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties + echo "PROJECT_NAME=$GERRIT_PROJECT" >> $WORK_DIRECTORY/scenario.properties exit 0 fi @@ -70,6 +72,9 @@ function override_generic_scenario() { echo "Recording the installer '$INSTALLER_TYPE' and scenario '$DEPLOY_SCENARIO' for downstream jobs" echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties + echo "XCI_SHA=$XCI_SHA" >> $WORK_DIRECTORY/scenario.properties + echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties + echo "PROJECT_NAME=$GERRIT_PROJECT" >> $WORK_DIRECTORY/scenario.properties exit 0 fi else @@ -108,6 +113,12 @@ function determine_generic_scenario() { ;; esac done + + # extract releng-xci sha + XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha which is same as releng-xci sha for generic scenarios + SCENARIO_SHA=$XCI_SHA } # This function determines the impacted external scenario by processing the Gerrit @@ -134,6 +145,12 @@ function determine_external_scenario() { # process the diff to find out what scenario(s) are impacted - there should only be 1 DEPLOY_SCENARIO+=$(git diff HEAD^..HEAD --name-only | grep scenarios | awk -F '[/|/]' '{print $2}' | uniq) + + # extract releng-xci sha + XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha + SCENARIO_SHA=$(cd $WORK_DIRECTORY/$GERRIT_PROJECT && git rev-parse HEAD) } echo "Determining the impacted scenario" @@ -179,9 +196,17 @@ esac # save the installer and scenario names into java properties file # so they can be injected to downstream jobs via envInject -echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' for downstream jobs" +echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' and SHAs for downstream jobs" echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties +echo "XCI_SHA=$XCI_SHA" >> $WORK_DIRECTORY/scenario.properties +echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties +echo "PROJECT_NAME=$GERRIT_PROJECT" >> $WORK_DIRECTORY/scenario.properties + +# skip scenario support check if the job is promotion job +if [[ "$JOB_NAME" =~ (os|k8) ]]; then + exit 0 +fi # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml diff --git a/jjb/xci/xci-start-new-vm.sh b/jjb/xci/xci-start-new-vm.sh index 7874e6776..dd953e927 100755 --- a/jjb/xci/xci-start-new-vm.sh +++ b/jjb/xci/xci-start-new-vm.sh @@ -63,6 +63,9 @@ export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ export INSTALLER_TYPE=$INSTALLER_TYPE export GIT_BASE=$GIT_BASE export JENKINS_HOME=$JENKINS_HOME +export CI_LOOP=$CI_LOOP +export BUILD_TAG=$BUILD_TAG +export NODE_NAME=$NODE_NAME if [[ ! -z ${WORKSPACE+x} && $GERRIT_PROJECT != "releng-xci" ]]; then git clone https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT xci/scenarios/$DEPLOY_SCENARIO && cd xci/scenarios/$DEPLOY_SCENARIO diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 4c9ff9e69..e5c1a9394 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -66,7 +66,7 @@ use-build-blocker: true blocking-jobs: - 'xci-verify-{distro}-.*' - - 'xci-.*-{distro}-merge-.*' + - 'xci-merge-{distro}-.*' - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' @@ -110,6 +110,12 @@ forbidden-file-paths: - compare-type: ANT pattern: 'xci/scripts/vm/**' + - compare-type: ANT + pattern: 'docs/**' + - compare-type: ANT + pattern: 'prototypes/**' + - compare-type: ANT + pattern: 'upstream/**' - project-compare-type: 'REG_EXP' project-pattern: 'sfc|sdnvpn' branches: @@ -141,6 +147,15 @@ name: CLEAN_DIB_IMAGES default: 'true' - string: + name: CI_LOOP + default: 'verify' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' + - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' @@ -165,6 +180,7 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + CI_LOOP=$CI_LOOP node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -182,8 +198,9 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true kill-phase-on: NEVER abort-all-job: true @@ -223,6 +240,9 @@ name: DISTRO default: 'ubuntu' - string: + name: CI_LOOP + default: 'verify' + - string: name: FUNCTEST_MODE default: 'tier' - string: diff --git a/jjb/yardstick/yardstick-daily-jobs.yml b/jjb/yardstick/yardstick-daily-jobs.yml index 20890fa13..24c7f319b 100644 --- a/jjb/yardstick/yardstick-daily-jobs.yml +++ b/jjb/yardstick/yardstick-daily-jobs.yml @@ -262,8 +262,7 @@ - description-setter: description: "POD: $NODE_NAME" - 'yardstick-cleanup' - - 'yardstick-fetch-os-creds' - - 'yardstick-fetch-k8s-conf' + - 'yardstick-fetch-creds' - 'yardstick-{testsuite}' - 'yardstick-store-results' @@ -282,16 +281,23 @@ !include-raw: ./yardstick-daily.sh - builder: - name: yardstick-fetch-os-creds + name: yardstick-fetch-creds builders: - - shell: - !include-raw: ../../utils/fetch_os_creds.sh - -- builder: - name: yardstick-fetch-k8s-conf - builders: - - shell: - !include-raw: ./yardstick-get-k8s-conf.sh + # yamllint disable rule:indentation + - conditional-step: + condition-kind: regex-match + regex: "os-.*" + label: '$DEPLOY_SCENARIO' + steps: + - shell: + !include-raw: ../../utils/fetch_os_creds.sh + - conditional-step: + condition-kind: regex-match + regex: "k8-.*" + label: '$DEPLOY_SCENARIO' + steps: + - shell: + !include-raw: ./yardstick-get-k8s-conf.sh - builder: name: yardstick-store-results diff --git a/jjb/yardstick/yardstick-get-k8s-conf.sh b/jjb/yardstick/yardstick-get-k8s-conf.sh index 22a6aa4e6..116d2f0d0 100755 --- a/jjb/yardstick/yardstick-get-k8s-conf.sh +++ b/jjb/yardstick/yardstick-get-k8s-conf.sh @@ -7,6 +7,7 @@ if [[ "${DEPLOY_SCENARIO:0:2}" == "k8" ]];then if [[ ${INSTALLER_TYPE} == 'joid' ]];then juju scp kubernetes-master/0:config "${dest_path}" elif [[ ${INSTALLER_TYPE} == 'compass' ]];then + echo "Copy admin.conf to ${dest_path}" docker cp compass-tasks:/opt/admin.conf "${dest_path}" fi fi |