diff options
43 files changed, 419 insertions, 361 deletions
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index d6bb48555..74f67ce5f 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -128,7 +128,7 @@ if [[ "$JOB_NAME" =~ "virtual" ]]; then if [[ "$JOB_NAME" == *csit* ]]; then DEPLOY_CMD="${DEPLOY_CMD} -e csit-environment.yaml" fi - if [[ "$JOB_NAME" == *promote* ]]; then + if [[ "$PROMOTE" == "True" ]]; then DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 2" fi else diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh index f25325129..206c627ec 100755 --- a/jjb/apex/apex-download-artifact.sh +++ b/jjb/apex/apex-download-artifact.sh @@ -23,14 +23,13 @@ if [[ "$ARTIFACT_VERSION" =~ dev ]]; then tar -xvf apex-${OPNFV_ARTIFACT_VERSION}.tar.gz popd > /dev/null else - echo "Will download RPMs and ISO..." + echo "Will download RPMs..." # Must be RPMs/ISO - export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d") - echo "Downloading opnfv-${OPNFV_ARTIFACT_VERSION}.properties" + echo "Downloading latest properties file" # get the properties file in order to get info regarding artifacts - curl --fail -s -o $BUILD_DIRECTORY/opnfv.properties http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties + curl --fail -s -o $BUILD_DIRECTORY/opnfv.properties http://$GS_URL/latest.properties # source the file so we get OPNFV vars source $BUILD_DIRECTORY/opnfv.properties @@ -56,15 +55,6 @@ else echo "Unable to install new RPMs: $RPM_LIST" exit 1 fi - - # log info to console - echo "Downloading the ISO artifact using URL http://$OPNFV_ARTIFACT_URL" - echo "--------------------------------------------------------" - echo - - # Download ISO - curl --fail -s -o $BUILD_DIRECTORY/apex.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 - fi # TODO: Uncomment these lines to verify SHA512SUMs once the sums are diff --git a/jjb/apex/apex-snapshot-create.sh b/jjb/apex/apex-snapshot-create.sh index b2a39449e..342896c7d 100644 --- a/jjb/apex/apex-snapshot-create.sh +++ b/jjb/apex/apex-snapshot-create.sh @@ -13,7 +13,11 @@ set -o nounset set -o pipefail SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error) -SNAP_TYPE=$(echo ${JOB_NAME} | sed -n 's/^apex-\(.\+\)-promote.*$/\1/p') + +if [ -z "$SNAP_TYPE" ]; then + echo "ERROR: SNAP_TYPE not provided...exiting" + exit 1 +fi echo "Creating Apex snapshot..." echo "-------------------------" diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index 9d0b0148c..25870bb35 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -75,7 +75,10 @@ uploadrpm () { uploadsnap () { # Uploads snapshot artifact and updated properties file echo "Uploading snapshot artifacts" - SNAP_TYPE=$(echo ${JOB_NAME} | sed -n 's/^apex-\(.\+\)-promote.*$/\1/p') + if [ -z "$SNAP_TYPE" ]; then + echo "ERROR: SNAP_TYPE not provided...exiting" + exit 1 + fi gsutil cp $WORKSPACE/apex-${SNAP_TYPE}-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log if [ "$SNAP_TYPE" == 'csit' ]; then gsutil cp $WORKSPACE/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 861b713a0..50502d77a 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -26,6 +26,7 @@ virtual-slave: 'apex-virtual-master' baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' + concurrent-builds: 3 - danube: branch: 'stable/danube' @@ -34,6 +35,7 @@ virtual-slave: 'apex-virtual-danube' baremetal-slave: 'apex-baremetal-danube' verify-scenario: 'os-odl_l3-nofeature-ha' + concurrent-builds: 1 disabled: false platform: @@ -145,13 +147,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-daily.*' - - 'apex-deploy.*' - - 'apex-runner.*' - throttle: max-per-node: 3 max-total: 10 @@ -209,23 +204,23 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true - - multijob: - name: functest-smoke - condition: SUCCESSFUL - projects: - - name: 'functest-apex-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - DEPLOY_SCENARIO={verify-scenario} - FUNCTEST_SUITE_NAME=healthcheck - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: false +# - multijob: +# name: functest-smoke +# condition: SUCCESSFUL +# projects: +# - name: 'functest-apex-virtual-suite-{stream}' +# current-parameters: false +# predefined-parameters: | +# DEPLOY_SCENARIO={verify-scenario} +# FUNCTEST_SUITE_NAME=healthcheck +# GERRIT_BRANCH=$GERRIT_BRANCH +# GERRIT_REFSPEC=$GERRIT_REFSPEC +# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER +# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE +# node-parameters: true +# kill-phase-on: FAILURE +# abort-all-job: true +# git-revision: false # Verify Scenario Gate - job-template: @@ -279,13 +274,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-daily.*' - - 'apex-deploy.*' - - 'apex-runner.*' - throttle: max-per-node: 3 max-total: 10 @@ -310,23 +298,23 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true - - multijob: - name: functest-smoke - condition: SUCCESSFUL - projects: - - name: 'functest-apex-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - DEPLOY_SCENARIO={verify-scenario} - FUNCTEST_SUITE_NAME=healthcheck - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: false +# - multijob: +# name: functest-smoke +# condition: SUCCESSFUL +# projects: +# - name: 'functest-apex-virtual-suite-{stream}' +# current-parameters: false +# predefined-parameters: | +# DEPLOY_SCENARIO={verify-scenario} +# FUNCTEST_SUITE_NAME=healthcheck +# GERRIT_BRANCH=$GERRIT_BRANCH +# GERRIT_REFSPEC=$GERRIT_REFSPEC +# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER +# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE +# node-parameters: true +# kill-phase-on: FAILURE +# abort-all-job: true +# git-revision: false - job-template: name: 'apex-runner-cperf-{stream}' @@ -374,16 +362,18 @@ node-parameters: false current-parameters: true predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC= OPNFV_CLEAN=yes DEPLOY_SCENARIO={verify-scenario} kill-phase-on: FAILURE abort-all-job: true git-revision: false - multijob: - name: Functest - condition: ALWAYS + name: CPERF + condition: SUCCESSFUL projects: - - name: 'functest-apex-baremetal-daily-{stream}' + - name: 'cperf-apex-intel-pod2-daily-master' node-parameters: true current-parameters: false predefined-parameters: @@ -420,15 +410,20 @@ wrappers: - timeout: - timeout: 90 + timeout: 150 fail: true properties: - logrotate-default - throttle: - max-per-node: 3 + max-per-node: {concurrent-builds} max-total: 10 option: 'project' + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-verify-iso-{stream}' builders: - 'apex-build' @@ -467,11 +462,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-deploy.*' - throttle: max-per-node: 1 max-total: 10 @@ -593,6 +583,9 @@ current-parameters: true predefined-parameters: | OPNFV_CLEAN=yes + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC= + DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: FAILURE abort-all-job: true git-revision: false @@ -806,7 +799,7 @@ node-parameters: false current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l2-netvirt_gbp_fdio-noha + DEPLOY_SCENARIO=os-odl_netvirt-fdio-noha OPNFV_CLEAN=yes kill-phase-on: NEVER abort-all-job: true @@ -1161,6 +1154,7 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + PROMOTE=True node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -1187,7 +1181,9 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=csit node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -1197,7 +1193,9 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=csit node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -1259,12 +1257,13 @@ - name: 'apex-deploy-virtual-{stream}' current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l2-netvirt_gbp_fdio-noha + DEPLOY_SCENARIO=os-odl_netvirt-fdio-noha OPNFV_CLEAN=yes GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + PROMOTE=True node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -1274,7 +1273,9 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=fdio node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -1284,7 +1285,9 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=fdio node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -1324,6 +1327,10 @@ name: GS_URL default: $GS_BASE{gs-pathname} description: "URL to Google Storage." + - string: + name: PROMOTE + default: 'False' + description: "Flag to know if we should promote/upload snapshot artifacts." ######################## # builder macros diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index 1df6e154c..84a899f57 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -26,6 +26,7 @@ virtual-slave: 'apex-virtual-master' baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' + concurrent-builds: 3 - danube: branch: 'stable/danube' @@ -34,6 +35,7 @@ virtual-slave: 'apex-virtual-danube' baremetal-slave: 'apex-baremetal-danube' verify-scenario: 'os-odl_l3-nofeature-ha' + concurrent-builds: 1 disabled: false platform: @@ -145,13 +147,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-daily.*' - - 'apex-deploy.*' - - 'apex-runner.*' - throttle: max-per-node: 3 max-total: 10 @@ -209,23 +204,23 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true - - multijob: - name: functest-smoke - condition: SUCCESSFUL - projects: - - name: 'functest-apex-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - DEPLOY_SCENARIO={verify-scenario} - FUNCTEST_SUITE_NAME=healthcheck - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: false +# - multijob: +# name: functest-smoke +# condition: SUCCESSFUL +# projects: +# - name: 'functest-apex-virtual-suite-{stream}' +# current-parameters: false +# predefined-parameters: | +# DEPLOY_SCENARIO={verify-scenario} +# FUNCTEST_SUITE_NAME=healthcheck +# GERRIT_BRANCH=$GERRIT_BRANCH +# GERRIT_REFSPEC=$GERRIT_REFSPEC +# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER +# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE +# node-parameters: true +# kill-phase-on: FAILURE +# abort-all-job: true +# git-revision: false # Verify Scenario Gate - job-template: @@ -279,13 +274,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-daily.*' - - 'apex-deploy.*' - - 'apex-runner.*' - throttle: max-per-node: 3 max-total: 10 @@ -310,23 +298,23 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true - - multijob: - name: functest-smoke - condition: SUCCESSFUL - projects: - - name: 'functest-apex-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - DEPLOY_SCENARIO={verify-scenario} - FUNCTEST_SUITE_NAME=healthcheck - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - git-revision: false +# - multijob: +# name: functest-smoke +# condition: SUCCESSFUL +# projects: +# - name: 'functest-apex-virtual-suite-{stream}' +# current-parameters: false +# predefined-parameters: | +# DEPLOY_SCENARIO={verify-scenario} +# FUNCTEST_SUITE_NAME=healthcheck +# GERRIT_BRANCH=$GERRIT_BRANCH +# GERRIT_REFSPEC=$GERRIT_REFSPEC +# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER +# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE +# node-parameters: true +# kill-phase-on: FAILURE +# abort-all-job: true +# git-revision: false - job-template: name: 'apex-runner-cperf-{stream}' @@ -374,16 +362,18 @@ node-parameters: false current-parameters: true predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC= OPNFV_CLEAN=yes DEPLOY_SCENARIO={verify-scenario} kill-phase-on: FAILURE abort-all-job: true git-revision: false - multijob: - name: Functest - condition: ALWAYS + name: CPERF + condition: SUCCESSFUL projects: - - name: 'functest-apex-baremetal-daily-{stream}' + - name: 'cperf-apex-intel-pod2-daily-master' node-parameters: true current-parameters: false predefined-parameters: @@ -420,15 +410,20 @@ wrappers: - timeout: - timeout: 90 + timeout: 150 fail: true properties: - logrotate-default - throttle: - max-per-node: 3 + max-per-node: {concurrent-builds} max-total: 10 option: 'project' + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-verify-iso-{stream}' builders: - 'apex-build' @@ -467,11 +462,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-deploy.*' - throttle: max-per-node: 1 max-total: 10 @@ -593,6 +583,9 @@ current-parameters: true predefined-parameters: | OPNFV_CLEAN=yes + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC= + DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: FAILURE abort-all-job: true git-revision: false @@ -800,6 +793,7 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + PROMOTE=True node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -826,7 +820,9 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=csit node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -836,7 +832,9 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=csit node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -898,12 +896,13 @@ - name: 'apex-deploy-virtual-{stream}' current-parameters: false predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l2-netvirt_gbp_fdio-noha + DEPLOY_SCENARIO=os-odl_netvirt-fdio-noha OPNFV_CLEAN=yes GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + PROMOTE=True node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -913,7 +912,9 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=fdio node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -923,36 +924,14 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: true + current-parameters: false + predefined-parameters: | + SNAP_TYPE=fdio node-parameters: true kill-phase-on: FAILURE abort-all-job: true git-revision: false -- job-template: - name: 'apex-gs-clean-{stream}' - - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - node: '{slave}' - - disabled: false - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - builders: - - 'apex-gs-clean' - - triggers: - - 'apex-gs-clean-{stream}' - ######################## # parameter macros ######################## @@ -987,6 +966,10 @@ name: GS_URL default: $GS_BASE{gs-pathname} description: "URL to Google Storage." + - string: + name: PROMOTE + default: 'False' + description: "Flag to know if we should promote/upload snapshot artifacts." ######################## # builder macros @@ -1029,12 +1012,6 @@ !include-raw: ./apex-download-artifact.sh - builder: - name: 'apex-gs-cleanup' - builders: - - shell: - !include-raw: ./apex-gs-cleanup.sh - -- builder: name: 'apex-deploy' builders: - shell: @@ -1051,7 +1028,4 @@ name: 'apex-danube' triggers: - timed: '0 12 * * *' -- trigger: - name: 'apex-gs-clean-{stream}' - triggers: - - timed: '0 2 * * *' + diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden index bff7d3b5a..dc9107a11 100644 --- a/jjb/apex/scenarios.yaml.hidden +++ b/jjb/apex/scenarios.yaml.hidden @@ -15,7 +15,7 @@ danube: - 'os-nosdn-kvm-noha' - 'os-odl_l2-fdio-noha' - 'os-odl_l2-fdio-ha' - - 'os-odl_l2-netvirt_gbp_fdio-noha' + - 'os-odl_netvirt-fdio-noha' - 'os-odl_l2-sfc-noha' - 'os-odl_l3-nofeature-noha' - 'os-odl_l3-nofeature-ha' diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 17d520419..1531ff144 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -182,23 +182,28 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.dovetail only master by now, not sync with A/B/C branches + # 1.dovetail only master, based on D release # 2.here the stream means the SUT stream, dovetail stream is defined in its own job - # 3.only debug testsuite here(includes 3 basic testcase, - # i.e. one tempest smoke ipv6, two vping from functest) + # 3.only proposed_tests testsuite here(refstack, ha, ipv6, bgpvpn) # 4.not used for release criteria or compliance, # only to debug the dovetail tool bugs with arm pods - - trigger-builds: - - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' + # 5.only run against scenario os-(nosdn|odl_l2)-(nofeature-bgpvpn)-ha + - conditional-step: + condition-kind: regex-match + regex: os-(nosdn|odl_l2)-(nofeature|bgpvpn)-ha + label: '{scenario}' + steps: + - trigger-builds: + - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' - job-template: name: '{installer}-deploy-{pod}-daily-{stream}' diff --git a/jjb/barometer/barometer.yml b/jjb/barometer/barometer.yml index 9ec30e809..68b8a04c0 100644 --- a/jjb/barometer/barometer.yml +++ b/jjb/barometer/barometer.yml @@ -126,21 +126,16 @@ disabled: '{obj:disabled}' - concurrent: true + concurrent: false properties: - logrotate-default - - throttle: - enabled: true - max-total: 3 - max-per-node: 2 - option: 'project' parameters: - project-parameter: project: '{project}' branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - 'opnfv-build-centos-defaults' scm: - git-scm @@ -151,7 +146,6 @@ builders: - shell: | pwd - cd src - ./install_build_deps.sh - make clobber - make + cd ci + ./install_dependencies.sh + ./build_rpm.sh diff --git a/jjb/bottlenecks/bottlenecks-run-suite.sh b/jjb/bottlenecks/bottlenecks-run-suite.sh index 0df659af2..e6f8d1ba5 100644 --- a/jjb/bottlenecks/bottlenecks-run-suite.sh +++ b/jjb/bottlenecks/bottlenecks-run-suite.sh @@ -2,6 +2,7 @@ #set -e [[ $GERRIT_REFSPEC_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" BOTTLENECKS_IMAGE=opnfv/bottlenecks +REPORT="True" if [[ $SUITE_NAME == rubbos || $SUITE_NAME == vstf ]]; then echo "Bottlenecks: to pull image $BOTTLENECKS_IMAGE:${DOCKER_TAG}" @@ -53,11 +54,11 @@ else echo "Bottlenecks: pulling tutum/influxdb for yardstick" docker pull tutum/influxdb:0.13 sleep 5 - docker exec bottleneckcompose_bottlenecks_1 python ${POSCA_SCRIPT}/run_posca.py testcase $TEST_CASE + docker exec bottleneckcompose_bottlenecks_1 python ${POSCA_SCRIPT}/run_posca.py testcase $TEST_CASE $REPORT elif [[ $SUITE_NAME == posca_stress_ping ]]; then TEST_CASE=posca_factor_ping sleep 5 - docker exec bottleneckcompose_bottlenecks_1 python ${POSCA_SCRIPT}/run_posca.py testcase $TEST_CASE + docker exec bottleneckcompose_bottlenecks_1 python ${POSCA_SCRIPT}/run_posca.py testcase $TEST_CASE $REPORT fi echo "Bottlenecks: cleaning up docker-compose images and dockers" diff --git a/jjb/securityaudit/opnfv-security-audit.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml index 732df8925..732df8925 100644 --- a/jjb/securityaudit/opnfv-security-audit.yml +++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml diff --git a/jjb/daisy4nfv/daisy-deploy.sh b/jjb/daisy4nfv/daisy-deploy.sh index b512e3f60..785f3a571 100755 --- a/jjb/daisy4nfv/daisy-deploy.sh +++ b/jjb/daisy4nfv/daisy-deploy.sh @@ -6,7 +6,7 @@ echo "--------------------------------------------------------" echo "This is $INSTALLER_TYPE deploy job!" echo "--------------------------------------------------------" -DEPLOY_SCENARIO=${DEPLOY_SCENARIO:-"os-nosdn-nofeature-ha"} +DEPLOY_SCENARIO=${DEPLOY_SCENARIO:-"os-nosdn-nofeature-noha"} BRIDGE=${BRIDGE:-pxebr} LAB_NAME=${NODE_NAME/-*} POD_NAME=${NODE_NAME/*-} @@ -29,7 +29,7 @@ git clone ssh://jenkins-zte@gerrit.opnfv.org:29418/securedlab --quiet \ cp -r securedlab/labs . DEPLOY_COMMAND="sudo ./ci/deploy/deploy.sh -b $BASE_DIR \ - -l $LAB_NAME -p $POD_NAME -B $BRIDGE" + -l $LAB_NAME -p $POD_NAME -B $BRIDGE -s $DEPLOY_SCENARIO" # log info to console echo """ diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index e631ee9b9..0a9d43d25 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -71,7 +71,7 @@ project: '{project}' branch: '{branch}' - 'opnfv-build-centos-defaults' - - 'daisy-defaults' + - '{installer}-defaults' - '{installer}-project-parameter': gs-pathname: '{gs-pathname}' @@ -157,7 +157,8 @@ - project-parameter: project: '{project}' branch: '{branch}' - - 'daisy-defaults' + - '{installer}-defaults' + - '{slave-label}-defaults' - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT @@ -165,8 +166,6 @@ - string: name: DEPLOY_SCENARIO default: 'os-nosdn-nofeature-ha' - - 'daisy-defaults' - - '{slave-label}-defaults' - '{installer}-project-parameter': gs-pathname: '{gs-pathname}' diff --git a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml index 9e7b867af..561ffbe24 100644 --- a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml @@ -54,6 +54,11 @@ enabled: true max-total: 4 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{alias}-merge-(master|danube)' + block-level: 'NODE' scm: - git-scm @@ -148,7 +153,7 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - '{alias}-merge-(master|danube)' + - '{alias}-merge-{phase}-.*' block-level: 'NODE' scm: @@ -164,6 +169,7 @@ - project-parameter: project: '{project}' branch: '{branch}' + - '{installer}-defaults' - '{slave-label}-defaults' - '{alias}-merge-defaults': gs-pathname: '{gs-pathname}' diff --git a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml index a0ec2ebd7..dff0ff0a4 100644 --- a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml @@ -47,8 +47,14 @@ enabled: true max-total: 4 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-merge-build-.*' + block-level: 'NODE' + scm: - - git-scm + - git-scm-gerrit wrappers: - ssh-agent-wrapper - timeout: @@ -103,8 +109,13 @@ name: unit condition: SUCCESSFUL projects: - - name: '{alias}-verify-{name}-{stream}' - current-parameters: true + - name: '{alias}-verify-unit-{stream}' + current-parameters: false + predefined-parameters: | + BRANCH=$BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: false kill-phase-on: FAILURE abort-all-job: true @@ -133,8 +144,15 @@ enabled: true max-total: 6 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-merge-build-.*' + - '{alias}-verify-build-.*' + block-level: 'NODE' + scm: - - git-scm + - git-scm-gerrit wrappers: - ssh-agent-wrapper - timeout: @@ -144,9 +162,11 @@ - project-parameter: project: '{project}' branch: '{branch}' + - '{installer}-defaults' - '{slave-label}-defaults' - '{alias}-verify-defaults': gs-pathname: '{gs-pathname}' + builders: - description-setter: description: "Built on $NODE_NAME" diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 3ad67b60d..dce7e5862 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -6,9 +6,19 @@ set -e [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" +DOVETAIL_HOME=${WORKSPACE}/cvp +if [ -d ${DOVETAIL_HOME} ]; then + sudo rm -rf ${DOVETAIL_HOME}/* +else + sudo mkdir -p ${DOVETAIL_HOME} +fi + +DOVETAIL_CONFIG=${DOVETAIL_HOME}/pre_config +sudo mkdir -p ${DOVETAIL_CONFIG} + sshkey="" # The path of openrc.sh is defined in fetch_os_creds.sh -OPENRC=$WORKSPACE/opnfv-openrc.sh +OPENRC=${DOVETAIL_CONFIG}/env_config.sh if [[ ${INSTALLER_TYPE} == 'apex' ]]; then instack_mac=$(sudo virsh domiflist undercloud | grep default | \ grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") @@ -37,14 +47,15 @@ releng_repo=${WORKSPACE}/releng git clone https://gerrit.opnfv.org/gerrit/releng ${releng_repo} >/dev/null if [[ ${INSTALLER_TYPE} != 'joid' ]]; then - ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} + sudo /bin/bash ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} fi if [[ -f $OPENRC ]]; then echo "INFO: openstack credentials path is $OPENRC" cat $OPENRC else - echo "ERROR: file $OPENRC does not exist." + echo "ERROR: cannot find file $OPENRC. Please check if it is existing." + sudo ls -al ${DOVETAIL_CONFIG} exit 1 fi @@ -65,13 +76,8 @@ else echo "HA test cases may not run properly." fi -pod_file_dir="/home/opnfv/dovetail/userconfig" -if [ -d ${pod_file_dir} ]; then - sudo rm -rf ${pod_file_dir}/* -else - sudo mkdir -p ${pod_file_dir} -fi -cmd="sudo python ${releng_repo}/utils/create_pod_file.py -t ${INSTALLER_TYPE} -i ${INSTALLER_IP} ${options} -f ${pod_file_dir}/pod.yaml" +cmd="sudo python ${releng_repo}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \ + -i ${INSTALLER_IP} ${options} -f ${DOVETAIL_CONFIG}/pod.yaml" echo ${cmd} ${cmd} @@ -79,11 +85,12 @@ deactivate cd ${WORKSPACE} -if [ -f ${pod_file_dir}/pod.yaml ]; then - echo "file ${pod_file_dir}/pod.yaml:" - cat ${pod_file_dir}/pod.yaml +if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then + echo "file ${DOVETAIL_CONFIG}/pod.yaml:" + cat ${DOVETAIL_CONFIG}/pod.yaml else - echo "Error: There doesn't exist file ${pod_file_dir}/pod.yaml." + echo "Error: cannot find file ${DOVETAIL_CONFIG}/pod.yaml. Please check if it is existing." + sudo ls -al ${DOVETAIL_CONFIG} echo "HA test cases may not run properly." fi @@ -91,20 +98,22 @@ ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" if [ "$INSTALLER_TYPE" == "fuel" ]; then echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." - sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${pod_file_dir}/id_rsa + sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa fi +# sdnvpn test case needs to download this image first before running +sudo wget -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${DOVETAIL_CONFIG} + opts="--privileged=true -id" -results_envs="-v /var/run/docker.sock:/var/run/docker.sock \ - -v /home/opnfv/dovetail/results:/home/opnfv/dovetail/results" -openrc_volume="-v ${OPENRC}:${OPENRC}" -userconfig_volume="-v ${pod_file_dir}:${pod_file_dir}" + +docker_volume="-v /var/run/docker.sock:/var/run/docker.sock" +dovetail_home_volume="-v ${DOVETAIL_HOME}:${DOVETAIL_HOME}" # Pull the image with correct tag echo "Dovetail: Pulling image opnfv/dovetail:${DOCKER_TAG}" docker pull opnfv/dovetail:$DOCKER_TAG >$redirect -cmd="docker run ${opts} ${results_envs} ${openrc_volume} ${userconfig_volume} \ +cmd="docker run ${opts} -e DOVETAIL_HOME=${DOVETAIL_HOME} ${docker_volume} ${dovetail_home_volume} \ ${sshkey} opnfv/dovetail:${DOCKER_TAG} /bin/bash" echo "Dovetail: running docker run command: ${cmd}" ${cmd} >${redirect} @@ -126,13 +135,13 @@ if [ $(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | wc -l) == 0 ]; then fi list_cmd="dovetail list ${TESTSUITE}" -run_cmd="dovetail run --openrc ${OPENRC} --testsuite ${TESTSUITE} -d" +run_cmd="dovetail run --testsuite ${TESTSUITE} -d" echo "Container exec command: ${list_cmd}" docker exec $container_id ${list_cmd} echo "Container exec command: ${run_cmd}" docker exec $container_id ${run_cmd} -sudo cp -r ${DOVETAIL_REPO_DIR}/results ./ +sudo cp -r ${DOVETAIL_HOME}/results ./ # To make sure the file owner is the current user, for the copied results files in the above line # if not, there will be error when next time to wipe workspace # CURRENT_USER=${SUDO_USER:-$USER} diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh index e1a4c0267..2c0d12a80 100755 --- a/jjb/fuel/fuel-build.sh +++ b/jjb/fuel/fuel-build.sh @@ -7,6 +7,15 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## + +# disable Fuel iso build for master branch +if [[ "$BRANCH" == 'master' ]]; then + touch $WORKSPACE/.noupload + echo "--------------------------------------------------------" + echo "Done!" + exit 0 +fi + set -o errexit set -o nounset set -o pipefail diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 5432c9447..dd0590c72 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -303,19 +303,19 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 20 * * *' + - timed: '' # '5 20 * * *' - trigger: name: 'fuel-os-odl_l2-nofeature-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 23 * * *' + - timed: '' # '5 23 * * *' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 2 * * *' + - timed: '' # '5 2 * * *' - trigger: name: 'fuel-os-nosdn-ovs-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 5 * * *' + - timed: '' # '5 5 * * *' - trigger: name: 'fuel-os-onos-sfc-ha-baremetal-daily-master-trigger' triggers: @@ -327,23 +327,23 @@ - trigger: name: 'fuel-os-odl_l2-sfc-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 11 * * *' + - timed: '' # '5 11 * * *' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 14 * * *' + - timed: '' # '5 14 * * *' - trigger: name: 'fuel-os-nosdn-kvm-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 17 * * *' + - timed: '' # '5 17 * * *' - trigger: name: 'fuel-os-nosdn-kvm_ovs_dpdk-ha-baremetal-daily-master-trigger' triggers: - - timed: '30 12 * * *' + - timed: '' # '30 12 * * *' - trigger: name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-ha-baremetal-daily-master-trigger' triggers: - - timed: '30 8 * * *' + - timed: '' # '30 8 * * *' # NOHA Scenarios - trigger: name: 'fuel-os-nosdn-nofeature-noha-baremetal-daily-master-trigger' @@ -537,11 +537,11 @@ - trigger: name: 'fuel-os-odl_l2-nofeature-noha-virtual-daily-master-trigger' triggers: - - timed: '35 15 * * *' + - timed: '' # '35 15 * * *' - trigger: name: 'fuel-os-odl_l3-nofeature-noha-virtual-daily-master-trigger' triggers: - - timed: '5 18 * * *' + - timed: '' # '5 18 * * *' - trigger: name: 'fuel-os-onos-sfc-noha-virtual-daily-master-trigger' triggers: @@ -553,27 +553,27 @@ - trigger: name: 'fuel-os-odl_l2-sfc-noha-virtual-daily-master-trigger' triggers: - - timed: '35 1 * * *' + - timed: '' # '35 1 * * *' - trigger: name: 'fuel-os-odl_l2-bgpvpn-noha-virtual-daily-master-trigger' triggers: - - timed: '5 4 * * *' + - timed: '' # '5 4 * * *' - trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-master-trigger' triggers: - - timed: '35 6 * * *' + - timed: '' # '35 6 * * *' - trigger: name: 'fuel-os-nosdn-ovs-noha-virtual-daily-master-trigger' triggers: - - timed: '5 9 * * *' + - timed: '' # '5 9 * * *' - trigger: name: 'fuel-os-nosdn-kvm_ovs_dpdk-noha-virtual-daily-master-trigger' triggers: - - timed: '30 16 * * *' + - timed: '' # '30 16 * * *' - trigger: name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-noha-virtual-daily-master-trigger' triggers: - - timed: '30 20 * * *' + - timed: '' # '30 20 * * *' #----------------------------------------------- # Triggers for job running on fuel-virtual against danube branch #----------------------------------------------- diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index f5bbd1818..4d48ee587 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -21,7 +21,7 @@ echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment" if [[ "$JOB_NAME" =~ "merge" ]]; then # set simplest scenario for virtual deploys to run for merges DEPLOY_SCENARIO="os-nosdn-nofeature-ha" -else +elif [[ "$BRANCH" != 'master' ]]; then # for none-merge deployments # checkout the commit that was used for building the downloaded artifact # to make sure the ISO and deployment mechanism uses same versions diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index b180f591b..8de092d29 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -356,6 +356,8 @@ - 'functest-cleanup' - 'set-functest-env' - 'functest-suite' + - 'functest-store-results' + - 'functest-exit' - builder: name: functest-daily @@ -374,7 +376,10 @@ name: set-functest-env builders: - shell: - !include-raw: ./set-functest-env.sh + !include-raw: + - ./functest-env-presetup.sh + - ../../utils/fetch_os_creds.sh + - ./set-functest-env.sh - builder: name: functest-store-results diff --git a/jjb/functest/functest-env-presetup.sh b/jjb/functest/functest-env-presetup.sh new file mode 100755 index 000000000..7a9b09d2c --- /dev/null +++ b/jjb/functest/functest-env-presetup.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +# Fetch INSTALLER_IP for APEX deployments +if [[ ${INSTALLER_TYPE} == 'apex' ]]; then + echo "Gathering IP information for Apex installer VM" + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + if sudo virsh list | grep undercloud; then + echo "Installer VM detected" + undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + export INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) + export sshkey_vol="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" + sudo scp $ssh_options root@${INSTALLER_IP}:/home/stack/stackrc ${HOME}/stackrc + export stackrc_vol="-v ${HOME}/stackrc:/home/opnfv/functest/conf/stackrc" + + if sudo iptables -C FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then + sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable + fi + if sudo iptables -C FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then + sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable + fi + echo "Installer ip is ${INSTALLER_IP}" + else + echo "No available installer VM exists and no credentials provided...exiting" + exit 1 + fi +fi + diff --git a/jjb/functest/functest-suite.sh b/jjb/functest/functest-suite.sh index 228cc3da4..5d1ed28f5 100755 --- a/jjb/functest/functest-suite.sh +++ b/jjb/functest/functest-suite.sh @@ -15,4 +15,7 @@ for test in ${tests[@]}; do let global_ret_val+=$? done -exit $global_ret_val +ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value" +echo ${global_ret_val}>${ret_val_file} + +exit 0 diff --git a/jjb/functest/functest-weekly-jobs.yml b/jjb/functest/functest-weekly-jobs.yml index f44f7b8aa..59d24cc87 100644 --- a/jjb/functest/functest-weekly-jobs.yml +++ b/jjb/functest/functest-weekly-jobs.yml @@ -115,7 +115,10 @@ - shell: !include-raw: ./functest-cleanup.sh - shell: - !include-raw: ./set-functest-env.sh + !include-raw: + - ./functest-env-presetup.sh + - ../../utils/fetch_os_creds.sh + - ./set-functest-env.sh - shell: !include-raw: ./functest-loop.sh - shell: diff --git a/jjb/functest/set-functest-env.sh b/jjb/functest/set-functest-env.sh index 89dabb056..558e2487d 100755 --- a/jjb/functest/set-functest-env.sh +++ b/jjb/functest/set-functest-env.sh @@ -1,50 +1,16 @@ #!/bin/bash set -e -[[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" -# LAB_CONFIG is used only for joid +set +u +set +o pipefail +[[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" +# Prepare OpenStack credentials volume if [[ ${INSTALLER_TYPE} == 'joid' ]]; then - # If production lab then creds may be retrieved dynamically - # creds are on the jumphost, always in the same folder rc_file_vol="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" - # If dev lab, credentials may not be the default ones, just provide a path to put them into docker - # replace the default one by the customized one provided by jenkins config -fi - -if [[ ${RC_FILE_PATH} != '' ]] && [[ -f ${RC_FILE_PATH} ]] ; then - echo "Credentials file detected: ${RC_FILE_PATH}" - # volume if credentials file path is given to Functest - rc_file_vol="-v ${RC_FILE_PATH}:/home/opnfv/functest/conf/openstack.creds" - RC_FLAG=1 -fi - - -if [[ ${INSTALLER_TYPE} == 'apex' ]]; then - ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - if sudo virsh list | grep undercloud; then - echo "Installer VM detected" - undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ - grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") - INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) - sshkey_vol="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" - sudo scp $ssh_options root@${INSTALLER_IP}:/home/stack/stackrc ${HOME}/stackrc - stackrc_vol="-v ${HOME}/stackrc:/home/opnfv/functest/conf/stackrc" - - if sudo iptables -C FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then - sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable - fi - if sudo iptables -C FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then - sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable - fi - elif [[ "$RC_FLAG" == 1 ]]; then - echo "No available installer VM, but credentials provided...continuing" - else - echo "No available installer VM exists and no credentials provided...exiting" - exit 1 - fi - +else + rc_file_vol="-v ${HOME}/opnfv-openrc.sh:/home/opnfv/functest/conf/openstack.creds" fi @@ -58,11 +24,13 @@ DEPLOY_TYPE=baremetal echo "Functest: Start Docker and prepare environment" -echo "Functest: Download images that will be used by test cases" -images_dir="${HOME}/opnfv/functest/images" -chmod +x ${WORKSPACE}/functest/ci/download_images.sh -${WORKSPACE}/functest/ci/download_images.sh ${images_dir} -images_vol="-v ${images_dir}:/home/opnfv/functest/images" +if [ "$BRANCH" != 'stable/danube' ]; then + echo "Functest: Download images that will be used by test cases" + images_dir="${HOME}/opnfv/functest/images" + chmod +x ${WORKSPACE}/functest/ci/download_images.sh + ${WORKSPACE}/functest/ci/download_images.sh ${images_dir} 2> ${redirect} + images_vol="-v ${images_dir}:/home/opnfv/functest/images" +fi dir_result="${HOME}/opnfv/functest/results/${BRANCH##*/}" mkdir -p ${dir_result} @@ -85,7 +53,11 @@ if [[ ${INSTALLER_TYPE} == 'compass' && ${DEPLOY_SCENARIO} == *'os-nosdn-openo-h envs=${env}" -e OPENO_MSB_ENDPOINT=${openo_msb_endpoint}" fi -volumes="${images_vol} ${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" +if [ "$BRANCH" != 'stable/danube' ]; then + volumes="${images_vol} ${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" +else + volumes="${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" +fi HOST_ARCH=$(uname -m) FUNCTEST_IMAGE="opnfv/functest" diff --git a/jjb/global/installer-params.yml b/jjb/global/installer-params.yml index fc9f34a48..40fc42c76 100644 --- a/jjb/global/installer-params.yml +++ b/jjb/global/installer-params.yml @@ -95,12 +95,16 @@ parameters: - string: name: INSTALLER_IP - default: '10.20.0.2' + default: '10.20.7.3' description: 'IP of the installer' - string: name: INSTALLER_TYPE default: daisy description: 'Installer used for deploying OPNFV on this POD' + - string: + name: BRIDGE + default: 'br7' + description: 'pxe bridge for booting of Fuel master' - parameter: name: 'infra-defaults' diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 8c3131eb7..57442225c 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -225,6 +225,10 @@ name: SLAVE_LABEL default: 'daisy-baremetal' - string: + name: INSTALLER_IP + default: '10.20.11.2' + description: 'IP of the installer' + - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' @@ -324,6 +328,14 @@ name: SLAVE_LABEL default: 'daisy-virtual' - string: + name: INSTALLER_IP + default: '10.20.11.2' + description: 'IP of the installer' + - string: + name: BRIDGE + default: 'daisy1' + description: 'pxe bridge for booting of Fuel master' + - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' @@ -633,7 +645,7 @@ description: 'Git URL to use on this Jenkins Slave' - string: name: INSTALLER_IP - default: '10.20.7.2' + default: '10.20.7.3' description: 'IP of the installer' - string: name: BRIDGE diff --git a/jjb/qtip/helpers/validate-deploy.sh b/jjb/qtip/helpers/validate-deploy.sh index 9f3dbe414..af8f8c200 100644 --- a/jjb/qtip/helpers/validate-deploy.sh +++ b/jjb/qtip/helpers/validate-deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash ############################################################################## -# Copyright (c) 2016 ZTE and others. +# Copyright (c) 2017 ZTE and others. # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -8,40 +8,20 @@ ############################################################################## set -e -envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} --e NODE_NAME=${NODE_NAME} -e CI_DEBUG=${CI_DEBUG}" -ramfs=/tmp/qtip/ramfs -cfg_dir=$(dirname $ramfs) -dir_imgstore="${HOME}/imgstore" -ramfs_volume="$ramfs:/mnt/ramfs" - echo "--------------------------------------------------------" echo "POD: $NODE_NAME" -echo "INSTALLER: $INSTALLER_TYPE" echo "Scenario: $DEPLOY_SCENARIO" +echo "INSTALLER: $INSTALLER_TYPE" +echo "INSTALLER_IP: $INSTALLER_IP" echo "--------------------------------------------------------" echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}" -docker pull opnfv/qtip:$DOCKER_TAG - -# use ramfs to fix docker socket connection issue with overlay mode in centos -if [ ! -d $ramfs ]; then - mkdir -p $ramfs -fi +docker pull opnfv/qtip:$DOCKER_TAG >/dev/null -if [ ! -z "$(df $ramfs | tail -n -1 | grep $ramfs)" ]; then - sudo mount -t tmpfs -o size=32M tmpfs $ramfs -fi - -# enable contro path in docker -cat <<EOF > ${cfg_dir}/ansible.cfg -[defaults] -callback_whitelist = profile_tasks -[ssh_connection] -control_path=/mnt/ramfs/ansible-ssh-%%h-%%p-%%r -EOF +envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} +-e POD_NAME=${NODE_NAME} -e SCENARIO=${DEPLOY_SCENARIO}" -cmd=" docker run -id -e $envs -v ${ramfs_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash" +cmd=" docker run -id -e $envs opnfv/qtip:${DOCKER_TAG} /bin/bash" echo "Qtip: Running docker command: ${cmd}" ${cmd} @@ -49,14 +29,12 @@ container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..." exit 1 -else - echo "The container ID is: ${container_id}" - QTIP_REPO=/home/opnfv/repos/qtip - docker cp ${cfg_dir}/ansible.cfg ${container_id}:/home/opnfv/.ansible.cfg -# TODO(zhihui_wu): use qtip cli to execute benchmark test in the future - docker exec -t ${container_id} bash -c "cd ${QTIP_REPO}/qtip/runner/ && - python runner.py -d /home/opnfv/qtip/results/ -b all" - fi +echo "The container ID is: ${container_id}" +QTIP_REPO=/home/opnfv/repos/qtip + +docker exec -t ${container_id} bash -c "bash ${QTIP_REPO}/tests/ci/run_ci.sh" + echo "Qtip done!" +exit 0
\ No newline at end of file diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 3b7ec3478..5fe0eb913 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -25,6 +25,9 @@ project: # projects with jobs for master + - 'releng-anteater': + <<: *master + <<: *other-receivers - 'bottlenecks': <<: *master <<: *other-receivers diff --git a/jjb/storperf/storperf.yml b/jjb/storperf/storperf.yml index 709a1ebab..cd40050b2 100644 --- a/jjb/storperf/storperf.yml +++ b/jjb/storperf/storperf.yml @@ -13,10 +13,12 @@ branch: '{stream}' gs-pathname: '' disabled: false + docker-tag: 'latest' - danube: branch: 'stable/{stream}' gs-pathname: '/{stream}' disabled: false + docker-tag: 'stable' - job-template: name: 'storperf-verify-{stream}' @@ -149,6 +151,13 @@ project: '{project}' branch: '{branch}' - 'intel-pod9-defaults' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-noha' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' scm: - git-scm diff --git a/modules/requirements.txt b/modules/requirements.txt index 1eaf8d089..bae55dcd2 100644 --- a/modules/requirements.txt +++ b/modules/requirements.txt @@ -1,3 +1,3 @@ -paramiko>=2.0.1 +paramiko==2.1.2 mock==1.3.0 requests==2.9.1 diff --git a/prototypes/bifrost/scripts/bifrost-provision.sh b/prototypes/bifrost/scripts/bifrost-provision.sh index 056196254..2b9021528 100755 --- a/prototypes/bifrost/scripts/bifrost-provision.sh +++ b/prototypes/bifrost/scripts/bifrost-provision.sh @@ -57,8 +57,8 @@ WRITE_INTERFACES_FILE=true export DIB_DEV_USER_PWDLESS_SUDO=yes export DIB_DEV_USER_PASSWORD=devuser -# Settings for distro: trusty/ubuntu-minimal, 7/centos7, 42.2/suse -export DIB_OS_RELEASE=${DIB_OS_RELEASE:-trusty} +# Settings for distro: xenial/ubuntu-minimal, 7/centos7, 42.2/suse +export DIB_OS_RELEASE=${DIB_OS_RELEASE:-xenial} export DIB_OS_ELEMENT=${DIB_OS_ELEMENT:-ubuntu-minimal} # DIB OS packages diff --git a/prototypes/xci/README.rst b/prototypes/xci/README.rst index 6d7af0d6e..0d9366533 100644 --- a/prototypes/xci/README.rst +++ b/prototypes/xci/README.rst @@ -113,11 +113,17 @@ change into directory where the sandbox script is located execute sandbox script - sudo -E ./xci-deploy.sh + ./xci-deploy.sh Issuing above command will start aio sandbox deployment and the sandbox should be ready between 1,5 and 2 hours depending on the host machine. +Please remember that the user executing the XCI script will need to +have an ssh key available, and stored in $HOME/.ssh directory. +You can generate one by executing + + ssh-keygen -t rsa + Advanced Usage -------------- diff --git a/prototypes/xci/config/env-vars b/prototypes/xci/config/env-vars index cefb412a6..9d4c78258 100755 --- a/prototypes/xci/config/env-vars +++ b/prototypes/xci/config/env-vars @@ -6,7 +6,6 @@ export OPNFV_RELENG_GIT_URL=https://gerrit.opnfv.org/gerrit/releng.git export OPENSTACK_BIFROST_GIT_URL=https://git.openstack.org/openstack/bifrost export OPENSTACK_OSA_GIT_URL=https://git.openstack.org/openstack/openstack-ansible export OPENSTACK_OSA_ETC_PATH=/etc/openstack_deploy -export CLEAN_DIB_IMAGES=false export OPNFV_HOST_IP=192.168.122.2 export XCI_FLAVOR_ANSIBLE_FILE_PATH=$OPNFV_RELENG_PATH/prototypes/xci/file/$XCI_FLAVOR export CI_LOOP=${CI_LOOP:-daily} diff --git a/prototypes/xci/config/user-vars b/prototypes/xci/config/user-vars index f9de940a2..5ed539627 100755 --- a/prototypes/xci/config/user-vars +++ b/prototypes/xci/config/user-vars @@ -54,3 +54,5 @@ export OPNFV_OSA_PLAYBOOK=${OPNFV_OSA_PLAYBOOK:-"$OPENSTACK_OSA_PATH/playbooks/s export ANSIBLE_VERBOSITY=${ANSIBLE_VERBOSITY-""} export LOG_PATH=${LOG_PATH:-${XCI_DEVEL_ROOT}/opnfv/logs} export RUN_TEMPEST=${RUN_TEMPEST:-false} +# Set this to to true to force XCI to re-create the target OS images +export CLEAN_DIB_IMAGES=${CLEAN_DIB_IMAGES:-false} diff --git a/prototypes/xci/file/install-ansible.sh b/prototypes/xci/file/install-ansible.sh index daa7f516d..67a49b397 100644 --- a/prototypes/xci/file/install-ansible.sh +++ b/prototypes/xci/file/install-ansible.sh @@ -131,6 +131,6 @@ fi PIP=$(which pip) -sudo -H -E ${PIP} install "pip>6.0" +${PIP} install --user "pip>6.0" -pip install ansible==$XCI_ANSIBLE_PIP_VERSION +${PIP} install --user --upgrade ansible==$XCI_ANSIBLE_PIP_VERSION @@ -3,7 +3,7 @@ from setuptools import setup setup( - name="opnfv", + name="releng", version="master", url="https://www.opnfv.org", ) diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index 6a382a56c..458bbda3b 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -7,7 +7,9 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - +set -o errexit +set -o nounset +set -o pipefail usage() { echo "usage: $0 [-v] -d <destination> -i <installer_type> -a <installer_ip>" >&2 @@ -157,8 +159,8 @@ elif [ "$installer_type" == "compass" ]; then sshpass -p root scp 2>/dev/null $ssh_options root@${installer_ip}:~/admin-openrc.sh $dest_path &> /dev/null info "This file contains the mgmt keystone API, we need the public one for our rc file" - grep "OS_AUTH_URL.*v2" $dest_path > /dev/null 2>&1 - if [ $? -eq 0 ] ; then + + if grep "OS_AUTH_URL.*v2" $dest_path > /dev/null 2>&1 ; then public_ip=$(sshpass -p root ssh $ssh_options root@${installer_ip} \ "ssh ${controller_ip} 'source /opt/admin-openrc.sh; openstack endpoint show identity '" \ | grep publicurl | awk '{print $4}') @@ -223,5 +225,3 @@ fi echo "-------- Credentials: --------" cat $dest_path - -exit 0 diff --git a/utils/push-test-logs.sh b/utils/push-test-logs.sh index 9099657c8..f0c488a5a 100644 --- a/utils/push-test-logs.sh +++ b/utils/push-test-logs.sh @@ -22,6 +22,7 @@ dir_result="${HOME}/opnfv/$project/results/${branch}" # + intel-pod12 (vsperf) node_list=(\ 'lf-pod1' 'lf-pod2' 'intel-pod2' 'intel-pod12' \ +'lf-virtual2' 'lf-virtual3' \ 'intel-pod5' 'intel-pod6' 'intel-pod7' 'intel-pod8' \ 'ericsson-pod1' 'ericsson-pod2' \ 'ericsson-virtual1' 'ericsson-virtual2' 'ericsson-virtual3' \ diff --git a/utils/test/reporting/LICENSE.txt b/utils/test/reporting/LICENSE.txt index d447b560b..4d0b33c6e 100644 --- a/utils/test/reporting/LICENSE.txt +++ b/utils/test/reporting/LICENSE.txt @@ -1,5 +1,5 @@ -Creative Commons Attribution 3.0 Unported
-http://creativecommons.org/licenses/by/3.0/
+This work is licensed under a Creative Commons Attribution 4.0 International License.
+SPDX-License-Identifier: CC-BY-4.0
License
diff --git a/utils/test/reporting/README.txt b/utils/test/reporting/README.txt index 31b8d044d..049cf62ef 100644 --- a/utils/test/reporting/README.txt +++ b/utils/test/reporting/README.txt @@ -1,6 +1,6 @@ Phantom by HTML5 UP
html5up.net | @ajlkn
-Free for personal and commercial use under the CCA 3.0 license (html5up.net/license)
+SPDX-License-Identifier: CC-BY-4.0
This is Phantom, a simple design built around a grid of large, colorful, semi-interactive
@@ -30,4 +30,4 @@ Credits: html5shiv.js (@afarkas @jdalton @jon_neal @rem)
Misc. Sass functions (@HugoGiraudel)
Respond.js (j.mp/respondjs)
- Skel (skel.io)
\ No newline at end of file + Skel (skel.io)
diff --git a/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js b/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js index 3ce76c912..929f5433c 100644 --- a/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js +++ b/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js @@ -28,7 +28,7 @@ function raiseAlert($uibModal) { return function(mode, title, text) { $uibModal.open({ - templateUrl: '/shared/alerts/alertModal.html', + templateUrl: 'testapi-ui/shared/alerts/alertModal.html', controller: 'RaiseAlertModalController as alert', backdrop: true, keyboard: true, diff --git a/utils/test/testapi/opnfv_testapi/router/url_mappings.py b/utils/test/testapi/opnfv_testapi/router/url_mappings.py index aea85a4c4..a2312de09 100644 --- a/utils/test/testapi/opnfv_testapi/router/url_mappings.py +++ b/utils/test/testapi/opnfv_testapi/router/url_mappings.py @@ -55,7 +55,7 @@ mappings = [ (r"/api/v1/scenarios/([^/]+)", scenario_handlers.ScenarioGURHandler), # static path - (r'/(.*\.(css|png|gif|js|html|json|map))', + (r'/(.*\.(css|png|gif|js|html|json|map|woff2|woff|ttf))', tornado.web.StaticFileHandler, {'path': config.Config().static_path}), diff --git a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py index 73190841f..6a9d94eb2 100644 --- a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py +++ b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py @@ -32,6 +32,9 @@ class SigninHandler(base.BaseHandler): class SigninReturnHandler(base.BaseHandler): def get(self): + if self.get_query_argument(const.OPENID_MODE) == 'cancel': + self._auth_failure('Authentication canceled.') + openid = self.get_query_argument(const.OPENID_CLAIMED_ID) user_info = { 'openid': openid, @@ -44,6 +47,12 @@ class SigninReturnHandler(base.BaseHandler): self.set_secure_cookie('openid', openid) self.redirect(url=CONF.ui_url) + def _auth_failure(self, message): + params = {'message': message} + url = parse.urljoin(CONF.ui_url, + '/#/auth_failure?' + parse.urlencode(params)) + self.redirect(url) + class SignoutHandler(base.BaseHandler): def get(self): |