diff options
45 files changed, 1274 insertions, 534 deletions
@@ -2,7 +2,9 @@ extends: default rules: - # 120 chars should be enough and don't fail if a line is longer + # ONLY 'max' length is configure, other parameters are default values. line-length: max: 120 level: warning + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: false diff --git a/jjb/apex/apex-project-jobs.yml b/jjb/apex/apex-project-jobs.yml index b12b1015c..58072865d 100644 --- a/jjb/apex/apex-project-jobs.yml +++ b/jjb/apex/apex-project-jobs.yml @@ -46,10 +46,6 @@ branch: '{branch}' - apex-parameter: gs-pathname: '{gs-pathname}' - - string: - name: GERRIT_REFSPEC - default: 'refs/heads/{branch}' - description: "JJB configured GERRIT_REFSPEC parameter" scm: - git-scm-gerrit diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index b2fa62f87..da3992ea8 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -250,7 +250,7 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 16 * * 2,4,6' + - timed: '0 1 * * 2,4,6' - trigger: name: 'fuel-os-nosdn-nofeature-noha-armband-baremetal-master-trigger' triggers: @@ -258,14 +258,14 @@ - trigger: name: 'fuel-os-odl-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 16 * * 1,3,5,7' + - timed: '0 1 * * 1,3,5,7' # --------------------------------------------------------------------- # Enea Armband CI Baremetal Triggers running against euphrates branch # --------------------------------------------------------------------- - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-euphrates-trigger' triggers: - - timed: '0 1 * * 2,4,6' + - timed: '' - trigger: name: 'fuel-os-nosdn-nofeature-noha-armband-baremetal-euphrates-trigger' triggers: @@ -273,7 +273,7 @@ - trigger: name: 'fuel-os-odl-nofeature-ha-armband-baremetal-euphrates-trigger' triggers: - - timed: '0 1 * * 1,3,5,7' + - timed: '' # -------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against master branch # -------------------------------------------------------------- diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml index ef0da7f3a..56aee174c 100644 --- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml +++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml @@ -103,10 +103,6 @@ - project-parameter: project: '{repo}' branch: '{branch}' - - string: - name: GERRIT_REFSPEC - default: 'refs/heads/{stream}' - description: "Default Gerrit ref git HEAD should point to" scm: - git-scm-gerrit @@ -163,7 +159,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|fuel|functest|octopus|pharos|releng|sandbox|yardstick|infra|ipv6|kvmfornfv|lsoapi|models|moon|multisite|netready' + project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|copper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|fuel|functest|octopus|pharos|releng|sandbox|yardstick|infra|ipv6|kvmfornfv|lsoapi|models|moon|multisite|netready' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 0790b83ca..70296bda1 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -127,6 +127,7 @@ jobs: - 'compass-{scenario}-{pod}-daily-{stream}' - 'compass-deploy-{pod}-daily-{stream}' + - 'compass-collect-logs-{pod}-daily-{stream}' ######################## # job templates @@ -281,6 +282,25 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + - conditional-step: + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: master + label: '{stream}' + steps: + - trigger-builds: + - project: 'compass-collect-logs-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - job-template: name: 'compass-deploy-{pod}-daily-{stream}' @@ -346,6 +366,41 @@ - shell: !include-raw-escape: ./compass-deploy.sh +- job-template: + name: 'compass-collect-logs-{pod}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - logrotate-default + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - compass-ci-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + ppa-pathname: '{ppa-pathname}' + - '{slave-label}-defaults' + - '{installer}-defaults' + + scm: + - git-scm + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - fix-workspace-permissions + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - shell: + !include-raw-escape: ./compass-logs.sh + ######################## # parameter macros ######################## @@ -370,6 +425,10 @@ - string: name: PPA_CACHE default: "$WORKSPACE/work/repo/" + - string: + name: LOG_DIRECTORY + default: $WORKSPACE/log_output + description: "Directory where the logs will be located upon the completion of the collection." ######################## # trigger macros diff --git a/jjb/compass4nfv/compass-logs.sh b/jjb/compass4nfv/compass-logs.sh new file mode 100644 index 000000000..03c8b20ac --- /dev/null +++ b/jjb/compass4nfv/compass-logs.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -o nounset +set -o pipefail + +# log info to console +echo "Uploading the logs $INSTALLER_TYPE artifact. This could take some time..." +echo "--------------------------------------------------------" +echo + +# create the log directory if it doesn't exist +[[ -d $LOG_DIRECTORY ]] || mkdir -p $LOG_DIRECTORY + +OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +COMPASS_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_${OPNFV_ARTIFACT_VERSION}.log.tar.gz" + + +sudo docker exec -it compass-tasks /bin/bash /opt/collect-log.sh +sudo docker cp compass-tasks:/opt/log.tar.gz ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME} + +sudo chown $(whoami):$(whoami) ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME} + +gsutil cp "${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME}" \ + "gs://${GS_URL}/logs/${COMPASS_LOG_FILENAME}" > /dev/null 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available as http://${GS_URL}/logs/${COMPASS_LOG_FILENAME}" diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index 7024dad58..f215b78ac 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -136,11 +136,6 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - name: 'opnfv-yamllint-verify-{stream}' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - multijob: name: deploy-virtual condition: SUCCESSFUL @@ -261,11 +256,6 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - name: 'opnfv-yamllint-verify-{stream}' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - multijob: name: deploy-virtual condition: SUCCESSFUL @@ -302,7 +292,7 @@ wrappers: - ssh-agent-wrapper - timeout: - timeout: 240 + timeout: 360 fail: true - fix-workspace-permissions diff --git a/jjb/container4nfv/arm64/compass-build.sh b/jjb/container4nfv/arm64/compass-build.sh new file mode 100755 index 000000000..696f7ff76 --- /dev/null +++ b/jjb/container4nfv/arm64/compass-build.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +cd compass4nfv + +COMPASS_WORK_DIR=$WORKSPACE/../compass-work +mkdir -p $COMPASS_WORK_DIR +ln -s $COMPASS_WORK_DIR work + +#TODO: remove workaround after all arm64 patches merged +curl -s http://people.linaro.org/~yibo.cai/compass/compass4nfv-arm64-fixup.sh | bash -s {scenario} + +# build tarball +COMPASS_ISO_REPO='http://people.linaro.org/~yibo.cai/compass' ./build.sh diff --git a/jjb/container4nfv/arm64/compass-deploy.sh b/jjb/container4nfv/arm64/compass-deploy.sh new file mode 100755 index 000000000..3c599279e --- /dev/null +++ b/jjb/container4nfv/arm64/compass-deploy.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +cd compass4nfv + +export ADAPTER_OS_PATTERN='(?i)CentOS-7.*arm.*' +export OS_VERSION="centos7" +export KUBERNETES_VERSION="v1.7.3" +export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml" +export NETWORK="deploy/conf/vm_environment/network.yml" +export VIRT_NUMBER=2 VIRT_CPUS=2 VIRT_MEM=4096 VIRT_DISK=50G + +./deploy.sh diff --git a/jjb/container4nfv/container4nfv-arm64.yml b/jjb/container4nfv/container4nfv-arm64.yml new file mode 100644 index 000000000..16a10eac1 --- /dev/null +++ b/jjb/container4nfv/container4nfv-arm64.yml @@ -0,0 +1,52 @@ +--- + +- project: + name: 'container4nfv-arm64' + project: 'container4nfv' + installer: 'compass' + scenario: + - 'k8-multus-nofeature-noha': + disabled: false + - 'k8-sriov-nofeature-noha': + disabled: false + jobs: + - 'container4nfv-{scenario}-virtual-daily-master' + + +- job-template: + name: 'container4nfv-{scenario}-virtual-daily-master' + disabled: '{obj:disabled}' + concurrent: false + node: arm-packet01 + + scm: + - git: + url: https://gerrit.opnfv.org/gerrit/compass4nfv + branches: + - origin/master + basedir: compass4nfv + wipe-workspace: true + + triggers: + - 'trigger-{scenario}-virtual' + + wrappers: + - timeout: + timeout: 120 + fail: true + + builders: + - shell: + !include-raw: arm64/compass-build.sh + - shell: + !include-raw: arm64/compass-deploy.sh + + +- trigger: + name: 'trigger-k8-multus-nofeature-noha-virtual' + triggers: + - timed: '0 12 * * *' +- trigger: + name: 'trigger-k8-sriov-nofeature-noha-virtual' + triggers: + - timed: '0 16 * * *' diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index 3aa9505e7..6f4643ac8 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -129,15 +129,21 @@ - trigger-builds: - project: 'daisy-deploy-{pod}-daily-{stream}' current-parameters: false - predefined-parameters: + predefined-parameters: | DEPLOY_SCENARIO={scenario} + INSTALLER_VERSION={stream} + UPSTREAM_JOB_NAME=$JOB_NAME + UPSTREAM_BUILD_ID=$BUILD_ID same-node: true block: true - trigger-builds: - project: 'functest-daisy-{pod}-daily-{stream}' current-parameters: false - predefined-parameters: + predefined-parameters: | DEPLOY_SCENARIO={scenario} + INSTALLER_VERSION={stream} + UPSTREAM_JOB_NAME=$JOB_NAME + UPSTREAM_BUILD_ID=$BUILD_ID same-node: true block: true block-thresholds: @@ -157,8 +163,11 @@ - trigger-builds: - project: 'yardstick-daisy-{pod}-daily-{stream}' current-parameters: false - predefined-parameters: + predefined-parameters: | DEPLOY_SCENARIO={scenario} + INSTALLER_VERSION={stream} + UPSTREAM_JOB_NAME=$JOB_NAME + UPSTREAM_BUILD_ID=$BUILD_ID block: true same-node: true block-thresholds: @@ -214,10 +223,13 @@ builders: - description-setter: description: "POD: $NODE_NAME" + - 'track-begin-timestamp' - shell: !include-raw-escape: ./daisy4nfv-download-artifact.sh - shell: !include-raw-escape: ./daisy-deploy.sh + publishers: + - 'report-provision-result' ######################## # trigger macros diff --git a/jjb/daisy4nfv/daisy-deploy.sh b/jjb/daisy4nfv/daisy-deploy.sh index 1723fd109..f729c380e 100755 --- a/jjb/daisy4nfv/daisy-deploy.sh +++ b/jjb/daisy4nfv/daisy-deploy.sh @@ -30,7 +30,7 @@ fi LABS_DIR=/var/tmp/opnfv-${CONFIG_REPO_NAME} echo "Cloning ${CONFIG_REPO_NAME} repo $BRANCH to $LABS_DIR" -rm -rf $LABS_DIR +sudo rm -rf $LABS_DIR git clone ssh://jenkins-zte@gerrit.opnfv.org:29418/${CONFIG_REPO_NAME} \ --quiet --branch $BRANCH $LABS_DIR diff --git a/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh b/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh index 9a1e2fc20..0441ea159 100755 --- a/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh +++ b/jjb/daisy4nfv/daisy4nfv-build-kolla-image.sh @@ -14,35 +14,9 @@ set -o errexit set -o nounset set -o pipefail -importkey () { - # clone releng repository - echo "Cloning releng repository..." - [ -d releng ] && rm -rf releng - git clone https://gerrit.opnfv.org/gerrit/releng ./releng/ &> /dev/null - #this is where we import the siging key - if [ -f ./releng/utils/gpg_import_key.sh ]; then - source ./releng/utils/gpg_import_key.sh - fi -} - upload_image_to_opnfv () { image=$1 - importkey - if gpg2 --list-keys | grep "opnfv-helpdesk@rt.linuxfoundation.org"; then - echo "Signing Key avaliable" - SIGN_ARTIFACT="true" - fi - - if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then - gpg2 -vvv --batch --yes --no-tty \ - --default-key opnfv-helpdesk@rt.linuxfoundation.org \ - --passphrase besteffort \ - --detach-sig $image - gsutil cp $image.sig gs://$GS_URL/upstream/$image.sig - echo "Image signature upload complete!" - fi - sha512sum -b $image > $image.sha512sum gsutil cp $image.sha512sum gs://$GS_URL/upstream/$image.sha512sum diff --git a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh index a64c80e5c..ae5ca3813 100755 --- a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh +++ b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh @@ -68,7 +68,15 @@ else DOWNLOAD_CMD="curl -L -s -o $WORKSPACE/opnfv.bin" fi -$DOWNLOAD_CMD http://$OPNFV_ARTIFACT_URL > gsutil.bin.log 2>&1 +maxretries=3 +cnt=0 +rc=1 +while [ $cnt -lt $maxretries ] && [ $rc -ne 0 ] +do + cnt=$[cnt + 1] + $DOWNLOAD_CMD http://$OPNFV_ARTIFACT_URL > gsutil.bin.log 2>&1 + rc=$? +done # list the file ls -al $WORKSPACE/opnfv.bin diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 561240158..d8f43c9b2 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -17,19 +17,13 @@ disabled: false installer: - - apex: - slave-label: 'doctor-apex-verify' - - fuel: - slave-label: 'doctor-fuel-verify' - # - joid: - # slave-label: 'ool-virtual3' - # pod: 'ool-virtual3' + - 'apex' + - 'fuel' + # - 'joid' - phase: - - 'build-x86_64': - slave-label: 'doctor-{installer}-verify' - - 'build-aarch64': - slave-label: 'doctor-{installer}-{phase}-verify' + arch: + - 'x86_64' + - 'aarch64' inspector: - 'sample' @@ -41,37 +35,47 @@ exclude: - installer: 'apex' - phase: 'build-aarch64' + arch: 'aarch64' jobs: - 'doctor-verify-{stream}' - 'doctor-verify-unit-test-{stream}' - 'doctor-{task}-{inspector}-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{phase}-{stream}' + - 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' - job-template: name: 'doctor-verify-{stream}' - disabled: '{obj:disabled}' - project-type: 'multijob' - parameters: - project-parameter: project: '{project}' branch: '{branch}' - 'opnfv-build-ubuntu-defaults' - scm: - git-scm-gerrit - triggers: - - 'doctor-verify': - project: '{project}' - branch: '{branch}' - files: 'doctor_tests/**' - is-skip-vote: false - + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'doctor_tests/**' builders: - shell: | #!/bin/bash @@ -83,16 +87,12 @@ execution-type: PARALLEL projects: - name: 'doctor-verify-unit-test-{stream}' - current-parameters: false predefined-parameters: | GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - git-revision: true - node-parameters: false kill-phase-on: FAILURE - abort-all-job: false - job-template: name: 'doctor-verify-unit-test-{stream}' @@ -115,75 +115,15 @@ - job-template: name: 'doctor-{task}-{inspector}-{stream}' - disabled: '{obj:disabled}' - project-type: 'multijob' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - string: - name: OS_CREDS - default: /home/jenkins/openstack.creds - description: 'OpenStack credentials' - - string: - name: DOCKER_TAG - default: '{docker-tag}' - description: 'Tag to pull docker image' - - string: - name: CLEAN_DOCKER_IMAGES - default: 'false' - description: 'Remove downloaded docker images (opnfv/functest:*)' - - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' - description: 'Scenario to deploy and test' - # functest-suite-parameter - - string: - name: FUNCTEST_MODE - default: 'testcase' - - string: - name: FUNCTEST_SUITE_NAME - default: 'doctor-notification' - - string: - name: TESTCASE_OPTIONS - # yamllint disable rule:line-length - default: '-e INSPECTOR_TYPE={inspector} -v $WORKSPACE:/home/opnfv/repos/doctor' - # yamllint enable rule:line-length - description: 'Addtional parameters specific to test case(s)' - # functest-parameter - - string: - name: GS_PATHNAME - default: '{gs-pathname}' - # yamllint disable rule:line-length - description: "Version directory where the opnfv documents will be stored in gs repository" - # yamllint enable rule:line-length - - string: - name: FUNCTEST_REPO_DIR - default: "/home/opnfv/repos/functest" - description: "Directory where the Functest repository is cloned" - - string: - name: PUSH_RESULTS_TO_DB - default: "true" - description: "Push the results of all the tests to the resultDB" - - string: - name: CI_DEBUG - default: 'true' - description: "Show debug output information" - scm: - git-scm-gerrit - triggers: - '{auto-trigger-name}': project: '{project}' branch: '{branch}' files: 'doctor_tests/**' - is-skip-vote: true - builders: - shell: | #!/bin/bash @@ -194,52 +134,40 @@ name: 'doctor-verify-apex-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-apex-{inspector}-build-x86_64-{stream}' - current-parameters: false + - name: 'doctor-{task}-apex-{inspector}-x86_64-{stream}' predefined-parameters: | GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - git-revision: true - node-parameters: false kill-phase-on: FAILURE - abort-all-job: false - multijob: name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-build-x86_64-{stream}' - current-parameters: false + - name: 'doctor-{task}-fuel-{inspector}-x86_64-{stream}' predefined-parameters: | GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - git-revision: true - node-parameters: false kill-phase-on: FAILURE - abort-all-job: false - multijob: name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-build-aarch64-{stream}' - current-parameters: false + - name: 'doctor-{task}-fuel-{inspector}-aarch64-{stream}' predefined-parameters: | GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - git-revision: true - node-parameters: false kill-phase-on: FAILURE - abort-all-job: false - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{phase}-{stream}' + name: 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' disabled: '{obj:disabled}' - node: '{slave-label}' + node: 'doctor-{installer}-{arch}' wrappers: - ssh-agent-wrapper - build-timeout: @@ -249,13 +177,75 @@ project: '{project}' branch: '{branch}' - '{installer}-defaults' - - '{slave-label}-defaults' + - 'doctor-slave-parameter' + - 'doctor-parameter' + - 'doctor-functest-parameter' scm: - git-scm-gerrit builders: - 'doctor-verify-installer-inspector-builders-macro' publishers: - 'doctor-verify-publishers-macro' + + +# ------------------------------- +# parameter macros +# ------------------------------- +- parameter: + name: 'doctor-parameter' + parameters: + - string: + name: OS_CREDS + default: /home/jenkins/openstack.creds + description: 'OpenStack credentials' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' + - string: + name: CLEAN_DOCKER_IMAGES + default: 'false' + description: 'Remove downloaded docker images (opnfv/functest:*)' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + description: 'Scenario to deploy and test' + +- parameter: + name: 'doctor-functest-parameter' + parameters: + # functest-suite-parameter + - string: + name: FUNCTEST_MODE + default: 'testcase' + - string: + name: FUNCTEST_SUITE_NAME + default: 'doctor-notification' + - string: + name: TESTCASE_OPTIONS + # yamllint disable rule:line-length + default: '-e INSPECTOR_TYPE={inspector} -v $WORKSPACE:/home/opnfv/repos/doctor' + # yamllint enable rule:line-length + description: 'Addtional parameters specific to test case(s)' + # functest-parameter + - string: + name: GS_PATHNAME + default: '{gs-pathname}' + # yamllint disable rule:line-length + description: "Version directory where the opnfv documents will be stored in gs repository" + # yamllint enable rule:line-length + - string: + name: FUNCTEST_REPO_DIR + default: "/home/opnfv/repos/functest" + description: "Directory where the Functest repository is cloned" + - string: + name: PUSH_RESULTS_TO_DB + default: "true" + description: "Push the results of all the tests to the resultDB" + - string: + name: CI_DEBUG + default: 'true' + description: "Show debug output information" # ------------------------------- # builder macros # ------------------------------- @@ -338,7 +328,7 @@ - compare-type: ANT pattern: '{files}' skip-vote: - successful: '{is-skip-vote}' - failed: '{is-skip-vote}' - unstable: '{is-skip-vote}' - notbuilt: '{is-skip-vote}' + successful: true + failed: true + unstable: true + notbuilt: true diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 902e75499..1cb29d091 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -50,9 +50,6 @@ - zte-pod1: slave-label: zte-pod1 <<: *master - - zte-pod1: - slave-label: zte-pod1 - <<: *euphrates # ------------------------------- # scenarios # ------------------------------- @@ -631,7 +628,7 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-zte-pod1-daily-master-trigger' triggers: - - timed: '0 22 * * *' + - timed: '0 10 * * *' - trigger: name: 'fuel-os-odl-nofeature-ha-zte-pod1-daily-master-trigger' triggers: @@ -697,75 +694,3 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-noha-zte-pod1-daily-master-trigger' triggers: - timed: '' -# ---------------------------------------------- -# ZTE POD1 Triggers running against euphrates branch -# ---------------------------------------------- -- trigger: - name: 'fuel-os-nosdn-nofeature-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '0 10 * * *' -- trigger: - name: 'fuel-os-odl-nofeature-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-onos-sfc-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-onos-nofeature-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-kvm-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-ovs-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-kvm_ovs_dpdk-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -# NOHA Scenarios -- trigger: - name: 'fuel-os-nosdn-nofeature-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl-nofeature-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-onos-sfc-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-onos-nofeature-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-ovn-nofeature-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-kvm-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-ovs-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-kvm_ovs_dpdk-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index 3a52d1d50..35bf3dc01 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -38,7 +38,6 @@ fi # set deployment parameters export TMPDIR=${HOME}/tmpdir -BRIDGE=${BRIDGE:-pxebr} # shellcheck disable=SC2153 LAB_NAME=${NODE_NAME/-*} # shellcheck disable=SC2153 @@ -49,10 +48,6 @@ LAB_CONFIG_URL=${LAB_CONFIG_URL:-'ssh://jenkins-ericsson@gerrit.opnfv.org:29418/ # Fuel requires deploy script to be ran with sudo, Armband does not SUDO='sudo -E' if [ "${PROJECT}" = 'fuel' ]; then - # Fuel does not use any POD-specific configuration for virtual deploys - if [[ "${NODE_NAME}" =~ "virtual" ]]; then - POD_NAME="virtual_kvm" - fi # Fuel currently supports ericsson, intel, lf and zte labs if [[ ! "${LAB_NAME}" =~ (ericsson|intel|lf|zte) ]]; then echo "Unsupported/unidentified lab ${LAB_NAME}. Cannot continue!" @@ -74,21 +69,26 @@ mkdir -p "${TMPDIR}" chmod a+x "${HOME}" "${TMPDIR}" cd "${WORKSPACE}" || exit 1 -if [[ "${LAB_CONFIG_URL}" =~ ^(git|ssh):// ]]; then - echo "Cloning securedlab repo ${BRANCH}" - LOCAL_CFG="${TMPDIR}/securedlab" - rm -rf "${LOCAL_CFG}" - git clone --quiet --branch "${BRANCH}" "${LAB_CONFIG_URL}" "${LOCAL_CFG}" - LAB_CONFIG_URL="file://${LOCAL_CFG}" +if [[ "$BRANCH" =~ (danube|euphrates) ]]; then + if [[ "${LAB_CONFIG_URL}" =~ ^(git|ssh):// ]]; then + echo "Cloning securedlab repo ${BRANCH}" + LOCAL_CFG="${TMPDIR}/securedlab" + rm -rf "${LOCAL_CFG}" + git clone --quiet --branch "${BRANCH}" "${LAB_CONFIG_URL}" "${LOCAL_CFG}" + LAB_CONFIG_ARG="-b file://${LOCAL_CFG}" + BRIDGE_ARG="-B ${BRIDGE:-pxebr}" + else + LAB_CONFIG_ARG="-b ${LAB_CONFIG_URL}" + fi fi # log file name FUEL_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}.log.tar.gz" # construct the command -DEPLOY_COMMAND="${SUDO} ${WORKSPACE}/ci/deploy.sh -b ${LAB_CONFIG_URL} \ +DEPLOY_COMMAND="${SUDO} ${WORKSPACE}/ci/deploy.sh ${LAB_CONFIG_ARG:-} \ -l ${LAB_NAME} -p ${POD_NAME} -s ${DEPLOY_SCENARIO} ${ISO_FILE_ARG:-} \ - -B ${DEFAULT_BRIDGE:-${BRIDGE}} -S ${TMPDIR} \ + -S ${TMPDIR} ${BRIDGE_ARG:-} \ -L ${WORKSPACE}/${FUEL_LOG_FILENAME}" # log info to console diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index 735ca6f7f..b5db2ed37 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -6,7 +6,10 @@ set +o pipefail check_os_deployment() { FUNCTEST_IMAGE=opnfv/functest-healthcheck:${DOCKER_TAG} + echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." + docker pull ${FUNCTEST_IMAGE}>/dev/null cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} check_deployment" + echo "Checking deployment, CMD: ${cmd}" eval ${cmd} ret_value=$? if [ ${ret_value} != 0 ]; then @@ -107,8 +110,12 @@ elif [[ ${INSTALLER_TYPE} == 'compass' ]]; then elif [[ ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then cacert_file_vol="-v ${HOME}/os_cacert:/etc/ssl/certs/mcp_os_cacert" fi -rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/openstack.creds" +if [[ ${BRANCH} == "stable/euphrates" ]]; then + rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/openstack.creds" +else + rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/env_file" +fi # Set iptables rule to allow forwarding return traffic for container if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index 79e5c1503..1cf6892da 100644..100755 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -181,6 +181,11 @@ slave-label: '{pod}' installer: daisy <<: *euphrates + - zte-pod9: + slave-label: '{pod}' + installer: daisy + <<: *master + # ------------------------------- # PODs for verify jobs triggered by each patch upload # - ool-virtual1: # slave-label: '{pod}' @@ -244,13 +249,32 @@ builders: - description-setter: description: "Built on $NODE_NAME" - - 'functest-{testsuite}-builder' + - conditional-step: + condition-kind: regex-match + regex: "os-.*" + label: 'DEPLOY_SCENARIO' + steps: + - 'functest-{testsuite}-builder' + - conditional-step: + condition-kind: regex-match + regex: "k8-.*" + label: 'DEPLOY_SCENARIO' + steps: + - 'functest-k8-builder' ######################## # parameter macros ######################## - parameter: + name: functest-k8-parameter + parameters: + - string: + name: FUNCTEST_MODE + default: 'daily' + description: "Daily suite name to run on K8 deployment" + +- parameter: name: functest-daily-parameter parameters: - string: @@ -375,6 +399,23 @@ - 'functest-exit' - builder: + name: functest-k8-builder + builders: + - 'functest-cleanup' + - 'functest-daily-k8' + - 'functest-store-results' + - 'functest-exit' + +- builder: + name: functest-daily-k8 + builders: + # yamllint disable rule:indentation + - shell: + !include-raw: + - ../../utils/fetch_k8_conf.sh + - ./functest-k8.sh + +- builder: name: functest-daily builders: # yamllint disable rule:indentation diff --git a/jjb/functest/functest-k8.sh b/jjb/functest/functest-k8.sh new file mode 100755 index 000000000..6df5c53b0 --- /dev/null +++ b/jjb/functest/functest-k8.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e +set +u +set +o pipefail + +[[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" +FUNCTEST_DIR=/home/opnfv/functest + +rc_file=${HOME}/k8.creds +sudo rm -rf $rc_file + +if [[ ${INSTALLER_TYPE} == 'compass' ]]; then + admin_conf_file_vol = "-v ${HOME}/admin.conf:/root/.kube/config" + echo "export KUBECONFIG=/root/.kube/config" >> $rc_file + echo "export KUBERNETES_PROVIDER=local" >> $rc_file + KUBE_MASTER_URL = $(cat ${HOME}/admin.conf|grep server| awk '{print $2}') + echo "export KUBE_MASTER_URL=$KUBE_MASTER_URL" >> $rc_file + KUBE_MASTER_IP = $(echo $KUBE_MASTER_URL|awk -F'https://|:[0-9]+' '$0=$2') + echo "export KUBE_MASTER_IP=$KUBE_MASTER_IP" >> $rc_file +else + echo "Not supported by other installers yet" + exit 1 +fi + +rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/env_file" + +dir_result="${HOME}/opnfv/functest/results/${BRANCH##*/}" +mkdir -p ${dir_result} +sudo rm -rf ${dir_result}/* +results_vol="-v ${dir_result}:${FUNCTEST_DIR}/results" + +volumes="${rc_file_vol} ${results_vol} ${admin_conf_file_vol}" + +# Set iptables rule to allow forwarding return traffic for container +if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then + sudo iptables -I FORWARD -j RETURN +fi + +envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} \ + -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ + -e BUILD_TAG=${BUILD_TAG} -e DEPLOY_TYPE=${DEPLOY_TYPE}" + +DOCKER_TAG=`[[ ${BRANCH##*/} == "master" ]] && echo "latest" || echo ${BRANCH##*/}` + +FUNCTEST_IMAGE=opnfv/functest-kubernetes:${DOCKER_TAG} +docker pull ${FUNCTEST_IMAGE}>/dev/null +cmd_opt="run_tests -r -t all" +cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" +echo "Running Functest k8s test cases, CMD: ${cmd}" +eval ${cmd} +ret_value=$? +if [ ${ret_value} != 0 ]; then + echo ${ret_value} > ${ret_val_file} +fi diff --git a/jjb/global/installer-params.yml b/jjb/global/installer-params.yml index 916db808d..683ef78ba 100644 --- a/jjb/global/installer-params.yml +++ b/jjb/global/installer-params.yml @@ -51,10 +51,6 @@ default: 'floating_net' description: 'external network for test' - string: - name: BRIDGE - default: 'pxebr' - description: 'Bridge(s) to be used by salt master' - - string: name: GS_URL default: '$GS_BASE{gs-pathname}' description: "URL to Google Storage." @@ -80,8 +76,8 @@ description: 'Model to deploy (os|k8)' - string: name: OS_RELEASE - default: 'ocata' - description: 'OpenStack release (mitaka|newton|ocata)' + default: 'pike' + description: 'OpenStack release (mitaka|ocata|pike)' - string: name: EXTERNAL_NETWORK default: ext-net diff --git a/jjb/global/installer-report.sh b/jjb/global/installer-report.sh new file mode 100755 index 000000000..e2fcfd6f6 --- /dev/null +++ b/jjb/global/installer-report.sh @@ -0,0 +1,25 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +source $WORKSPACE/installer_track.sh +echo """ + INSTALLER: $INSTALLER + INSTALLER_VERSION: $INSTALLER_VERSION + JOB_NAME: $JOB_NAME + BUILD_ID: $BUILD_ID + SENARIO: $DEPLOY_SCENARIO + UPSTREAM_JOB_NAME: $UPSTREAM_JOB_NAME: + UPSTREAM_BUILD_ID: $UPSTREAM_BUILD_ID + PROVISION_RESULT: $PROVISION_RESULT + TIMESTAMP_START: $TIMESTAMP_START + TIMESTAMP_END: `date '+%Y-%m-%d %H:%M:%S.%3N'` + POD_NAME: $NODE_NAME +""" + +# TODO call TestAPI to report installer provisoin result when API is ready diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index aeeb24610..bd9f97d3b 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -35,6 +35,10 @@ name: GERRIT_BRANCH default: '{branch}' description: "JJB configured GERRIT_BRANCH parameter (deprecated)" + - string: + name: GERRIT_REFSPEC + default: 'refs/heads/{branch}' + description: "Default refspec needed for manually triggering." - property: name: logrotate-default @@ -151,7 +155,7 @@ - gerrit: server-name: 'gerrit.opnfv.org' trigger-on: - - ref-updated + - ref-updated-event projects: - project-compare-type: 'ANT' project-pattern: '{project}' @@ -294,128 +298,97 @@ fi - builder: - name: lint-python-code + name: upload-review-docs + builders: + - upload-under-review-docs-to-opnfv-artifacts + - report-build-result-to-gerrit + +- builder: + name: lint-init builders: - shell: | #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - virtualenv -p python2.7 $WORKSPACE/releng_flake8 - source $WORKSPACE/releng_flake8/bin/activate - - # install python packages - pip install "flake8==2.6.2" - - # generate and upload lint log - echo "Running flake8 code on $PROJECT ..." - - # Get number of flake8 violations. If none, this will be an - # empty string: "" - FLAKE_COUNT="$(find . \ - -path './releng_flake8' -prune -o \ - -path './.tox' -prune -o \ - -type f -name "*.py" -print | \ - xargs flake8 --exit-zero -qq --count 2>&1)" - # Ensure we start with a clean environment - rm -f lint.log - - if [ ! -z $FLAKE_COUNT ]; then - echo "Flake8 Violations: $FLAKE_COUNT" > lint.log - find . \ - -path './releng_flake8' -prune -o \ - -path './.tox' -prune -o \ - -type f -name "*.py" -print | \ - xargs flake8 --exit-zero --first >> violation.log - SHOWN=$(wc -l violation.log | cut -d' ' -f1) - echo -e "First $SHOWN shown\n---" >> lint.log - cat violation.log >> lint.log - sed -r -i '4,$s/^/ /g' lint.log - rm violation.log - fi - - deactivate + rm -f bash-violation.log python-violation.log yaml-violation.log violation.log + git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 > modified_files - builder: - name: report-lint-result-to-gerrit + name: lint-report builders: - shell: | #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # If no violations were found, no lint log will exist. - if [[ -e lint.log ]] ; then - echo -e "\nposting linting report to gerrit...\n" - - cat lint.log - echo - - ssh -p 29418 gerrit.opnfv.org \ - "gerrit review -p $GERRIT_PROJECT \ - -m \"$(cat lint.log)\" \ - $GERRIT_PATCHSET_REVISION \ - --notify NONE" - + if [[ -s violation.log ]]; then + echo "Reporting lint result..." + msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset." + sed -i -e '1s/^//$msg\n\n/' violation.log + cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE" + ssh -p 29418 gerrit.opnfv.org "$cmd" + + # Make sure the caller job failed exit 1 fi - builder: - name: upload-review-docs + name: lint-bash-code builders: - - upload-under-review-docs-to-opnfv-artifacts - - report-build-result-to-gerrit + - shell: | + #!/bin/bash + echo "Checking bash code..." + for f in $(egrep '\.sh$' modified_files) + do + bash -n "$f" 2>> bash-violation.log + done + if [[ -s bash-violation.log ]]; then + echo -e "Bash syntax error(s)\n---" >> violation.log + sed -e 's/^/ /g' bash-violation.log >> violation.log + fi - builder: - name: check-bash-syntax + name: lint-python-code builders: - - shell: "find . -name '*.sh' | xargs bash -n" + - shell: | + #!/bin/bash + # Install python package + sudo pip install "flake8==2.6.2" + + echo "Checking python code..." + for f in $(egrep '\.py$' modified_files) + do + flake8 "$f" >> python-violation.log + done + if [[ -s python-violation.log ]]; then + echo -e "Python violation(s)\n---" >> violation.log + sed -e 's/^/ /g' python-violation.log >> violation.log + fi - builder: name: lint-yaml-code builders: - shell: | #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # install python packages + # sudo Install python packages sudo pip install "yamllint==1.8.2" - # generate and upload lint log - echo "Running yaml code on $PROJECT ..." - - # Get list of yaml files - YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true - - #If YAML_FILES is none exit with 0 - if [ -z "$YAML_FILES" ]; then - exit 0 - fi - - # Ensure we start with a clean environment - rm -f yaml-violation.log lint.log - - # Yamllint files only in patchset - for yamlfile in $YAML_FILES; do - yamllint $yamlfile >> yaml-violation.log || true + echo "Checking yaml file..." + for f in $(egrep '\.ya?ml$' modified_files) + do + yamllint "$f" >> yaml-violation.log done - - if [ -s "yaml-violation.log" ]; then - SHOWN=$(grep -c -v "^$" yaml-violation.log) - echo -e "First $SHOWN shown\n---" > lint.log - cat yaml-violation.log >> lint.log - sed -r -i '4,$s/^/ /g' lint.log + if [[ -s yaml-violation.log ]]; then + echo -e "YAML violation(s)\n---" >> violation.log + sed -e 's/^/ /g' yaml-violation.log >> violation.log fi - builder: + name: lint-all-code + builders: + - lint-init + - lint-bash-code + - lint-python-code + - lint-yaml-code + - lint-report + +- builder: name: clean-workspace builders: - shell: | @@ -431,6 +404,12 @@ - shell: | find $WORKSPACE -type f -name '*.log' | xargs rm -f +- builder: + name: track-begin-timestamp + builders: + - shell: | + echo "export TIMESTAMP_START="\'`date '+%Y-%m-%d %H:%M:%S.%3N'`\' > $WORKSPACE/installer_track.sh + - publisher: name: archive-artifacts publishers: @@ -971,3 +950,23 @@ <<: *email_ptl_defaults recipients: > ross.b.brattain@intel.com + +- publisher: + name: 'report-provision-result' + publishers: + - postbuildscript: + script-only-if-succeeded: true + builders: + - shell: | + echo "export PROVISION_RESULT=SUCCEED" >> $WORKSPACE/installer_track.sh + echo "export INSTALLER={installer}" >> $WORKSPACE/installer_track.sh + - shell: + !include-raw-escape: installer-report.sh + - postbuildscript: + script-only-if-failed: true + builders: + - shell: | + echo "export PROVISION_RESULT=FAIL" >> $WORKSPACE/installer_track.sh + echo "export INSTALLER={installer}" >> $WORKSPACE/installer_track.sh + - shell: + !include-raw-escape: installer-report.sh diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 5136e7148..4902dcfc5 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -178,6 +178,44 @@ default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' +- parameter: + name: 'lf-pod4-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - lf-pod4 + default-slaves: + - lf-pod4 + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: SSH_KEY + default: /root/.ssh/id_rsa + description: 'SSH key to use for Apex' + +- parameter: + name: 'lf-pod5-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - lf-pod5 + default-slaves: + - lf-pod5 + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: SSH_KEY + default: /root/.ssh/id_rsa + description: 'SSH key to use for Apex' + ##################################################### # Parameters for CI baremetal PODs ##################################################### @@ -980,15 +1018,19 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'doctor-defaults' + name: 'doctor-slave-parameter' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - '{default-slave}' + - 'doctor-apex-x86_64' + - 'doctor-fuel-x86_64' + - 'doctor-fuel-aarch64' + - 'doctor-joid-x86_64' + - 'not-specified' default-slaves: - - '{default-slave}' + - 'not-specified' - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT @@ -999,30 +1041,6 @@ description: 'SSH key to be used' - parameter: - name: 'doctor-apex-verify-defaults' - parameters: - - 'doctor-defaults': - default-slave: 'doctor-apex-verify' - -- parameter: - name: 'doctor-fuel-verify-defaults' - parameters: - - 'doctor-defaults': - default-slave: 'doctor-fuel-verify' - -- parameter: - name: 'doctor-fuel-build-aarch64-verify-defaults' - parameters: - - 'doctor-defaults': - default-slave: 'doctor-fuel-build-aarch64-verify' - -- parameter: - name: 'doctor-joid-verify-defaults' - parameters: - - 'doctor-defaults': - default-slave: 'doctor-joid-verify' - -- parameter: name: 'multisite-virtual-defaults' parameters: - label: diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml index 3dfb9d155..908ddba9d 100644 --- a/jjb/opnfvdocs/opnfvdocs.yml +++ b/jjb/opnfvdocs/opnfvdocs.yml @@ -69,7 +69,7 @@ notbuilt: true builders: - - check-bash-syntax + - lint-bash-code - job-template: name: 'opnfvdocs-merge-shellcheck-{stream}' @@ -107,4 +107,4 @@ branch-pattern: '**/{branch}' builders: - - check-bash-syntax + - lint-bash-code diff --git a/jjb/pharos/check-jinja2.yml b/jjb/pharos/check-jinja2.yml index 8ee2a2aa1..7251d3f61 100644 --- a/jjb/pharos/check-jinja2.yml +++ b/jjb/pharos/check-jinja2.yml @@ -71,17 +71,25 @@ pattern: 'config/installers/**/*.j2' - compare-type: ANT pattern: 'check-jinja2.sh' + - compare-type: ANT + pattern: 'check-schema.sh' skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true + successful: false + failed: false + unstable: false + notbuilt: false builders: - check-pdf-jinja + - check-pdf-schema - builder: name: check-pdf-jinja builders: - shell: | $WORKSPACE/config/utils/check-jinja2.sh +- builder: + name: check-pdf-schema + builders: + - shell: | + $WORKSPACE/config/utils/check-schema.sh diff --git a/jjb/pharos/pharos.yml b/jjb/pharos/pharos.yml index 9f4feb42e..acf14885e 100644 --- a/jjb/pharos/pharos.yml +++ b/jjb/pharos/pharos.yml @@ -6,9 +6,12 @@ - '{name}' - '{name}-tools' + disabled: false + jobs: - '{project}-verify-basic' - 'backup-pharos-dashboard' + - 'deploy-pharos-dashboard' - job-template: name: 'backup-pharos-dashboard' @@ -16,6 +19,33 @@ disabled: '{obj:disabled}' parameters: + - 'pharos-dashboard-defaults' + + triggers: + - timed: '@daily' + + builders: + - shell: | + BACKUP_DIR=$HOME/backups/ + TMP_DIR=$HOME/tmp/ + mkdir -p $BACKUP_DIR + echo "-- $(date +%Y%m%d) --" + echo "Backing up Pharos Dashboard data..." + sudo docker run --rm \ + -v pharos-data:/pharos-data:ro \ + -v $TMP_DIR:/backup \ + alpine \ + tar -czf /backup/pharos-dashboard-db-$(date +%Y%m%d).tar.gz -C /pharos-data ./ + sudo mv $TMP_DIR/pharos-dashboard-db-$(date +%Y%m%d).tar.gz $BACKUP_DIR + sudo chown $USER:$USER $BACKUP_DIR/pharos-dashboard-db-$(date +%Y%m%d).tar.gz + echo "...complete" + +- job-template: + name: 'deploy-pharos-dashboard' + + disabled: '{obj:disabled}' + + parameters: - project-parameter: project: 'pharos-tools' branch: 'master' @@ -25,19 +55,22 @@ - git-scm-gerrit triggers: - - timed: '@daily' + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - ref-updated-event + - comment-added-contains-event: + comment-contains-value: '^redeploy$' + projects: + - project-compare-type: 'ANT' + project-pattern: 'pharos-tools' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' builders: - shell: | - mkdir -p /home/backups/ - DATE=$(date +%Y%m%d) - echo "-- $DATE --" - echo "Backing up Pharos Dashboard data..." - sudo docker run -it --rm \ - -v pharos_data:/pharos_data:ro \ - -v /tmp:/backup \ - alpine \ - tar -czf /backup/pharos-dashboard-db-$(date +%Y%m%d).tar.gz -C /pharos_data ./ - sudo mv /tmp/pharos-dashboard-db-$(date +%Y%m%d).tar.gz /home/backups/ - sudo chown $USER:$USER $WORKSPACE/backups/pharos-dashboard-db-$(date +%Y%m%d).tar.gz - echo "...complete" + cp $HOME/config.env $WORKSPACE/dashboard + cd $WORKSPACE/dashboard + sudo docker-compose build + sudo docker-compose up -d diff --git a/jjb/qtip/qtip-verify-jobs.yml b/jjb/qtip/qtip-verify-jobs.yml index 783c92b04..8698fa055 100644 --- a/jjb/qtip/qtip-verify-jobs.yml +++ b/jjb/qtip/qtip-verify-jobs.yml @@ -122,10 +122,6 @@ name: GS_URL default: '$GS_BASE{gs-pathname}' description: "Directory where the build artifact will be located upon the completion of the build." - - string: - name: GERRIT_REFSPEC - default: 'refs/heads/{branch}' - description: "JJB configured GERRIT_REFSPEC parameter" scm: - git-scm diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml index afeb35821..6bd952a1d 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/automate.yml @@ -46,6 +46,12 @@ builders: - mongodb-backup + publishers: + - email: + recipients: serena.feng.711@gmail.com + notify-every-unstable-build: true + send-to-individuals: true + - job-template: name: '{module}-verify-{stream}' diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml index db2e4273b..7f142241c 100644 --- a/jjb/releng/compass4nfv-docker.yml +++ b/jjb/releng/compass4nfv-docker.yml @@ -36,7 +36,7 @@ predefined-parameters: | PUSH_IMAGE=$PUSH_IMAGE COMMIT_ID=$COMMIT_ID - RELEASE_VERSION=$RELEASE_VERSION + GERRIT_REFNAME=$GERRIT_REFNAME DOCKERFILE=$DOCKERFILE kill-phase-on: FAILURE abort-all-jobs: false @@ -146,9 +146,9 @@ default: "" description: "commit id to make a snapshot docker image" - string: - name: RELEASE_VERSION + name: GERRIT_REFNAME default: "" - description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" - string: name: DOCKERFILE default: "Dockerfile" diff --git a/jjb/releng/functest-docker.yml b/jjb/releng/functest-docker.yml index 92dd54eaf..584dc302c 100644 --- a/jjb/releng/functest-docker.yml +++ b/jjb/releng/functest-docker.yml @@ -41,7 +41,7 @@ predefined-parameters: | PUSH_IMAGE=$PUSH_IMAGE COMMIT_ID=$COMMIT_ID - RELEASE_VERSION=$RELEASE_VERSION + GERRIT_REFNAME=$GERRIT_REFNAME DOCKERFILE=$DOCKERFILE kill-phase-on: FAILURE abort-all-jobs: false @@ -51,7 +51,7 @@ git-revision: true node-parameters: false predefined-parameters: - RELEASE_VERSION=$RELEASE_VERSION + GERRIT_REFNAME=$GERRIT_REFNAME kill-phase-on: FAILURE abort-all-jobs: false @@ -89,6 +89,8 @@ triggers: - pollscm: cron: "*/30 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' builders: - multijob: @@ -132,8 +134,12 @@ <<: *build-job-settings - name: 'functest-vnf-docker-build-amd64-{stream}' <<: *build-job-settings + - name: 'functest-vnf-docker-build-arm64-{stream}' + <<: *build-job-settings - name: 'functest-restapi-docker-build-amd64-{stream}' <<: *build-job-settings + - name: 'functest-restapi-docker-build-arm64-{stream}' + <<: *build-job-settings - multijob: name: 'publish all manifests' condition: SUCCESSFUL @@ -196,10 +202,6 @@ default: "{project}" description: "Project name used to enable job conditions" - string: - name: RELEASE_VERSION - default: "" - description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" - - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' @@ -215,18 +217,10 @@ *) tag="{stream}" ;; esac - case "{image}" in - "vnf"|"restapi") - sudo manifest-tool push from-args \ - --platforms linux/amd64 \ - --template opnfv/functest-{image}:ARCH-$tag \ - --target opnfv/functest-{image}:$tag ;; - *) - sudo manifest-tool push from-args \ - --platforms linux/amd64,linux/arm64 \ - --template opnfv/functest-{image}:ARCH-$tag \ - --target opnfv/functest-{image}:$tag ;; - esac + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template opnfv/functest-{image}:ARCH-$tag \ + --target opnfv/functest-{image}:$tag exit $? # parameter macro @@ -253,9 +247,9 @@ default: "" description: "commit id to make a snapshot docker image" - string: - name: RELEASE_VERSION + name: GERRIT_REFNAME default: "" - description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" - string: name: DOCKERFILE default: "Dockerfile" diff --git a/jjb/releng/opnfv-docker-arm.yml b/jjb/releng/opnfv-docker-arm.yml index 75e41c1a7..fbf565844 100644 --- a/jjb/releng/opnfv-docker-arm.yml +++ b/jjb/releng/opnfv-docker-arm.yml @@ -154,9 +154,9 @@ default: "{docker_repo_name}" description: "Dockerhub repo to be pushed to." - string: - name: RELEASE_VERSION + name: GERRIT_REFNAME default: "" - description: "Release version, e.g. 1.0, 2.0, 3.0" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" - string: name: DOCKER_DIR default: "{dockerdir}" @@ -185,6 +185,8 @@ triggers: - pollscm: cron: "*/30 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' publishers: - email: diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index 7f646f1d9..28e5b7ce1 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -12,8 +12,8 @@ set -o nounset set -o pipefail - -echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..." +echo "Using Docker $(docker --version) on $NODE_NAME" +echo "Starting Docker build for $DOCKER_REPO_NAME ..." echo "--------------------------------------------------------" echo @@ -77,6 +77,9 @@ echo "Current branch: $BRANCH" BUILD_BRANCH=$BRANCH +GERRIT_REFNAME=${GERRIT_REFNAME:-''} +RELEASE_VERSION=${GERRIT_REFNAME/refs\/tags//} + if [[ "$BRANCH" == "master" ]]; then DOCKER_TAG="latest" elif [[ -n "${RELEASE_VERSION-}" ]]; then diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 355cb5df4..317efc807 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -39,34 +39,29 @@ project: 'releng-anteater' <<: *master <<: *other-receivers - - 'barometer': + - 'barometer-collectd': project: 'barometer' - dockerdir: '.' - dockerfile: 'docker/Dockerfile' + dockerdir: 'docker/barometer-collectd' <<: *master <<: *other-receivers - 'barometer-grafana': project: 'barometer' - dockerdir: '.' - dockerfile: 'docker/barometer-grafana/Dockerfile' + dockerdir: 'docker/barometer-grafana' <<: *master <<: *other-receivers - 'barometer-influxdb': project: 'barometer' - dockerdir: '.' - dockerfile: 'docker/barometer-influxdb/Dockerfile' + dockerdir: 'docker/barometer-influxdb' <<: *master <<: *other-receivers - 'barometer-kafka': project: 'barometer' - dockerdir: '.' - dockerfile: 'docker/barometer-kafka/Dockerfile' + dockerdir: 'docker/barometer-kafka' <<: *master <<: *other-receivers - 'barometer-ves': project: 'barometer' - dockerdir: '.' - dockerfile: 'docker/barometer-ves/Dockerfile' + dockerdir: 'docker/barometer-ves' <<: *master <<: *other-receivers - 'bottlenecks': @@ -125,6 +120,10 @@ project: 'yardstick' <<: *master <<: *other-receivers + - 'functest-kubernetes': + project: 'functest-kubernetes' + <<: *master + <<: *other-receivers # projects with jobs for euphrates - 'bottlenecks': project: 'bottlenecks' @@ -232,9 +231,9 @@ default: "" description: "commit id to make a snapshot docker image" - string: - name: RELEASE_VERSION + name: GERRIT_REFNAME default: "" - description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" - string: name: DOCKERFILE default: "{dockerfile}" @@ -259,6 +258,8 @@ triggers: - pollscm: cron: "*/30 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' publishers: - email: diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index d566430d6..e9e929de6 100644 --- a/jjb/releng/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -31,10 +31,73 @@ disabled: '{obj:disabled}' + concurrent: true + parameters: - project-parameter: project: $GERRIT_PROJECT branch: '{branch}' + - node: + name: SLAVE_NAME + description: Slaves to execute yamllint + default-slaves: + - lf-build1 + allowed-multiselect: true + ignore-offline-nodes: true + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: 'releng|doctor|pharos' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*.py' + - compare-type: ANT + pattern: '**/*.sh' + - compare-type: ANT + pattern: '**/*.yml' + - compare-type: ANT + pattern: '**/*.yaml' + + builders: + - lint-all-code + +- job-template: + name: 'opnfv-pylint-verify-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + branch: '{branch}' + - node: + name: SLAVE_NAME + description: Slaves to execute yamllint + default-slaves: + - lf-build1 + allowed-multiselect: true + ignore-offline-nodes: true scm: - git-scm-gerrit @@ -54,7 +117,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -63,8 +126,9 @@ pattern: '**/*.py' builders: + - lint-init - lint-python-code - - report-lint-result-to-gerrit + - lint-report - job-template: name: 'opnfv-yamllint-verify-{stream}' @@ -103,7 +167,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'armband|fuel|octopus|pharos|releng|releng-anteater' + project-pattern: 'armband|fuel|octopus|releng-anteater' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -114,5 +178,6 @@ pattern: '**/*.yaml' builders: + - lint-init - lint-yaml-code - - report-lint-result-to-gerrit + - lint-report diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index c4b7c8db3..ef99f5abb 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -90,6 +90,12 @@ project: '{project}' branch: 'master' + properties: + - throttle: + max-per-node: 1 + max-total: 1 + option: 'project' + scm: - git-scm diff --git a/jjb/snaps/snaps-verify-jobs.yml b/jjb/snaps/snaps-verify-jobs.yml index bc3f39315..a6c86c966 100644 --- a/jjb/snaps/snaps-verify-jobs.yml +++ b/jjb/snaps/snaps-verify-jobs.yml @@ -15,7 +15,7 @@ - master: branch: '{stream}' gs-pathname: '' - disabled: false + disabled: true - job-template: name: 'snaps-verify-{stream}' diff --git a/jjb/xci/xci-cleanup.sh b/jjb/xci/xci-cleanup.sh new file mode 100755 index 000000000..45b4ec273 --- /dev/null +++ b/jjb/xci/xci-cleanup.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# skip the deployment if the patch doesn't impact the deployment +if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then + echo "Skipping the deployment!" + exit 0 +fi + +sudo virsh destroy ${DISTRO}_xci_vm +sudo virsh undefine ${DISTRO}_xci_vm diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml new file mode 100644 index 000000000..351fe22bb --- /dev/null +++ b/jjb/xci/xci-merge-jobs.yml @@ -0,0 +1,257 @@ +--- +- project: + name: 'opnfv-xci-merge' + + # ------------------------------- + # branches + # ------------------------------- + stream: + - master: + branch: '{stream}' + # ------------------------------- + # distros + # ------------------------------- + distro: + - ubuntu: + disabled: false + - centos: + disabled: false + - opensuse: + disabled: false + # ------------------------------- + # type + # ------------------------------- + type: + - virtual + # ------------------------------- + # patch verification phases + # ------------------------------- + phase: + - 'deploy' + - 'healthcheck' + - 'promote' + # ------------------------------- + # scenarios + # ------------------------------- + scenario: + - 'os-odl-sfc': + project: sfc + # ------------------------------- + # jobs + # ------------------------------- + jobs: + - 'xci-{scenario}-{type}-{distro}-merge-{stream}' + - 'xci-{phase}-{type}-{distro}-merge-{stream}' +# ------------------------------- +# job templates +# ------------------------------- +- job-template: + name: 'xci-{scenario}-{type}-{distro}-merge-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-verify-{distro}-.*' + - 'bifrost-verify-{distro}-.*' + - 'bifrost-periodic-{distro}-.*' + - 'xci-osa-verify-{distro}-.*' + - 'xci-osa-periodic-{distro}-.*' + - 'xci-(os|k8s).*?-virtual-{distro}-.*' + block-level: 'NODE' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + scm: + - git: + credentials-id: '$SSH_CREDENTIAL_ID' + url: 'https://gerrit.opnfv.org/gerrit/releng-xci' + branches: + - 'origin/master' + timeout: 15 + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{obj:project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + file-paths: + - compare-type: ANT + pattern: 'scenarios/**' + readable-message: true + custom-url: '* $JOB_NAME $BUILD_URL' + + parameters: + - project-parameter: + project: '{obj:project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'xci-virtual' + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'xci-deploy-{type}-{distro}-merge-{stream}' + current-parameters: true + predefined-parameters: | + PROJECT=$PROJECT + DISTRO={distro} + DEPLOY_SCENARIO={scenario} + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: healthcheck + condition: SUCCESSFUL + projects: + - name: 'xci-healthcheck-{type}-{distro}-merge-{stream}' + current-parameters: true + predefined-parameters: | + PROJECT=$PROJECT + DISTRO={distro} + DEPLOY_SCENARIO={scenario} + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + FUNCTEST_MODE=tier + FUNCTEST_TIER=healthcheck + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + +- job-template: + name: 'xci-{phase}-{type}-{distro}-merge-{stream}' + + disabled: false + + concurrent: true + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '.*-bifrost-verify-.*' + - '.*-bifrost-periodic-.*' + - 'osa-verify-.*' + - 'osa-periodic-.*' + block-level: 'NODE' + + parameters: + - string: + name: PROJECT + default: 'releng-xci' + - string: + name: DISTRO + default: 'ubuntu' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-noha' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'healthcheck' + - string: + name: XCI_FLAVOR + default: 'mini' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - string: + name: OPNFV_RELENG_DEV_PATH + default: $WORKSPACE/ + - string: + name: INSTALLER_TYPE + default: 'osa' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + scm: + - git: + credentials-id: '$SSH_CREDENTIAL_ID' + url: 'https://gerrit.opnfv.org/gerrit/releng-xci' + branches: + - 'origin/master' + timeout: 15 + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - 'xci-merge-{phase}-macro' + +# ------------------------------- +# builder macros +# ------------------------------- +- builder: + name: 'xci-merge-deploy-macro' + builders: + - shell: + !include-raw: ./xci-start-new-vm.sh + - shell: + !include-raw: ./xci-start-deployment.sh + +- builder: + name: 'xci-merge-healthcheck-macro' + builders: + - shell: + !include-raw: ./xci-run-functest.sh + - shell: + !include-raw: ./xci-cleanup.sh + +- builder: + name: 'xci-merge-promote-macro' + builders: + - shell: + !include-raw: ./xci-promote.sh diff --git a/jjb/xci/xci-promote.sh b/jjb/xci/xci-promote.sh new file mode 100755 index 000000000..98ad7ff4a --- /dev/null +++ b/jjb/xci/xci-promote.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 Ericsson and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# skip the healthcheck if the patch doesn't impact the deployment +if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then + echo "Skipping the healthcheck!" + exit 0 +fi + +# skip the deployment if the scenario is not supported on this distro +OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml +if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then + echo "# SKIPPED: Scenario $DEPLOY_SCENARIO is NOT supported on $DISTRO" + exit 0 +fi + +echo "Hello World!" diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh new file mode 100755 index 000000000..b2421de32 --- /dev/null +++ b/jjb/xci/xci-run-functest.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# skip the healthcheck if the patch doesn't impact the deployment +if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then + echo "Skipping the healthcheck!" + exit 0 +fi + +# skip the deployment if the scenario is not supported on this distro +OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml +if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then + echo "# SKIPPED: Scenario $DEPLOY_SCENARIO is NOT supported on $DISTRO" + exit 0 +fi + +ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci/xci && PATH=/home/devuser/.local/bin:$PATH ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/run-functest.yml" diff --git a/jjb/xci/xci-start-deployment.sh b/jjb/xci/xci-start-deployment.sh new file mode 100755 index 000000000..e86f5b3a1 --- /dev/null +++ b/jjb/xci/xci-start-deployment.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# skip the deployment if the patch doesn't impact the deployment +if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then + echo "Skipping the deployment!" + exit 0 +fi + +# skip the deployment if the scenario is not supported on this distro +OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml +if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then + echo "# SKIPPED: Scenario $DEPLOY_SCENARIO is NOT supported on $DISTRO" + exit 0 +fi + +ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" diff --git a/jjb/xci/xci-start-new-vm.sh b/jjb/xci/xci-start-new-vm.sh new file mode 100755 index 000000000..b72c33920 --- /dev/null +++ b/jjb/xci/xci-start-new-vm.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#---------------------------------------------------------------------- +# This script is used by CI and executed by Jenkins jobs. +# You are not supposed to use this script manually if you don't know +# what you are doing. +#---------------------------------------------------------------------- + +# skip the deployment if the patch doesn't impact the deployment +if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then + echo "Skipping the deployment!" + exit 0 +fi + +# skip the deployment if the scenario is not supported on this distro +OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml +if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then + echo "# SKIPPED: Scenario $DEPLOY_SCENARIO is NOT supported on $DISTRO" + exit 0 +fi + +cd $WORKSPACE + +# The start-new-vm.sh script will copy the entire releng-xci directory +# so lets prepare the test script now so it can be copied by the script. +# Please do not move it elsewhere or you would have to move it to the VM +# yourself. +cat > xci_test.sh<<EOF +#!/bin/bash +export DISTRO=$DISTRO +export DEPLOY_SCENARIO=$DEPLOY_SCENARIO +export FUNCTEST_MODE=$FUNCTEST_MODE +export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME +export XCI_FLAVOR=$XCI_FLAVOR +export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES +export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ +export INSTALLER_TYPE=$INSTALLER_TYPE +export GIT_BASE=$GIT_BASE +export JENKINS_HOME=$JENKINS_HOME + +if [[ ! -z ${WORKSPACE+x} && $GERRIT_PROJECT != "releng-xci" ]]; then + git clone https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT xci/scenarios/$DEPLOY_SCENARIO && cd xci/scenarios/$DEPLOY_SCENARIO + git fetch https://gerrit.opnfv.org/gerrit/$GERRIT_PROJECT $GERRIT_REFSPEC && git checkout FETCH_HEAD + cd - +fi + +cd xci +./xci-deploy.sh +EOF +chmod a+x xci_test.sh + +export XCI_BUILD_CLEAN_VM_OS=false +export XCI_UPDATE_CLEAN_VM_OS=true + +./xci/scripts/vm/start-new-vm.sh $DISTRO diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index fffd5c2b4..28bc317bd 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -21,10 +21,10 @@ notbuilt: false - centos: disabled: false - successful: true - failed: true - unstable: true - notbuilt: true + successful: false + failed: false + unstable: false + notbuilt: false - opensuse: disabled: false successful: false @@ -244,85 +244,14 @@ - builder: name: 'xci-verify-deploy-macro' builders: - - shell: | - #!/bin/bash - - # skip the deployment if the patch doesn't impact the deployment - if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then - echo "Skipping the deployment!" - exit 0 - fi - - cd $WORKSPACE - - # The start-new-vm.sh script will copy the entire releng-xci directory - # so lets prepare the test script now so it can be copied by the script. - # Please do not move it elsewhere or you would have to move it to the VM - # yourself. - cat > xci_test.sh<<EOF - #!/bin/bash - export DISTRO=$DISTRO - export DEPLOY_SCENARIO=$DEPLOY_SCENARIO - export FUNCTEST_MODE=$FUNCTEST_MODE - export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME - export XCI_FLAVOR=$XCI_FLAVOR - export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES - export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ - export INSTALLER_TYPE=$INSTALLER_TYPE - export GIT_BASE=$GIT_BASE - export JENKINS_HOME=$JENKINS_HOME - - cd xci - ./xci-deploy.sh - EOF - chmod a+x xci_test.sh - - export XCI_BUILD_CLEAN_VM_OS=false - export XCI_UPDATE_CLEAN_VM_OS=true - - ./xci/scripts/vm/start-new-vm.sh $DISTRO - - shell: | - #!/bin/bash - - # skip the deployment if the patch doesn't impact the deployment - if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then - echo "Skipping the deployment!" - exit 0 - fi - - ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" + - shell: + !include-raw: ./xci-start-new-vm.sh + - shell: + !include-raw: ./xci-start-deployment.sh - builder: name: 'xci-verify-healthcheck-macro' builders: - - shell: | - #!/bin/bash - - # skip the healthcheck if the patch doesn't impact the deployment - if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then - echo "Skipping the healthcheck!" - exit 0 - fi - - echo "Hello World!" - - shell: | - #!/bin/bash - - # skip the deployment if the patch doesn't impact the deployment - if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then - echo "Skipping the deployment!" - exit 0 - fi - - sudo virsh destroy ${DISTRO}_xci_vm - sudo virsh undefine ${DISTRO}_xci_vm - -# this will be enabled once the xci is prepared -# - builder: -# name: 'xci-verify-healthcheck-macro' -# builders: -# - shell: -# !include-raw: ../../utils/fetch_os_creds.sh -# - shell: -# !include-raw: ../functest/functest-alpine.sh + - shell: + !include-raw: ./xci-cleanup.sh diff --git a/utils/fetch_k8_conf.sh b/utils/fetch_k8_conf.sh new file mode 100755 index 000000000..f82fa5497 --- /dev/null +++ b/utils/fetch_k8_conf.sh @@ -0,0 +1,63 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +info () { + logger -s -t "fetch_k8_conf.info" "$*" +} + + +error () { + logger -s -t "fetch_k8_conf.error" "$*" + exit 1 +} + +: ${DEPLOY_TYPE:=''} + +#Get options +while getopts ":d:i:a:h:s:o:v" optchar; do + case "${optchar}" in + d) dest_path=${OPTARG} ;; + i) installer_type=${OPTARG} ;; + v) DEPLOY_TYPE="virt" ;; + *) echo "Non-option argument: '-${OPTARG}'" >&2 + usage + exit 2 + ;; + esac +done + +# set vars from env if not provided by user as options +dest_path=${dest_path:-$HOME/admin.conf} +installer_type=${installer_type:-$INSTALLER_TYPE} + +if [ -z $dest_path ] || [ -z $installer_type ]; then + usage + exit 2 +fi + +# Checking if destination path is valid +if [ -d $dest_path ]; then + error "Please provide the full destination path for the credentials file including the filename" +else + # Check if we can create the file (e.g. path is correct) + touch $dest_path || error "Cannot create the file specified. Check that the path is correct and run the script again." +fi + +info "Fetching admin.conf file..." +if [ "$installer_type" == "compass" ]; then + sudo docker cp compass-tasks:/opt/admin.conf $dest_path &> /dev/null + sudo chown $(whoami):$(whoami) $dest_path + info "Fetch admin.conf successfully" +else + error "Installer $installer is not supported by this script" +fi + |