diff options
| -rw-r--r-- | .gitmodules | 3 | ||||
| -rwxr-xr-x | jjb/apex/apex-deploy.sh | 9 | ||||
| -rwxr-xr-x | jjb/apex/apex-download-artifact.sh | 5 | ||||
| -rw-r--r-- | jjb/container4nfv/yardstick-arm64.yml | 2 | ||||
| -rw-r--r-- | jjb/daisy4nfv/daisy-daily-jobs.yml | 6 | ||||
| -rw-r--r-- | jjb/daisy4nfv/daisy-project-jobs.yml | 7 | ||||
| -rw-r--r-- | jjb/doctor/doctor.yml | 39 | ||||
| -rwxr-xr-x | jjb/functest/functest-alpine.sh | 4 | ||||
| -rw-r--r-- | jjb/functest/functest-docker.yml (renamed from jjb/releng/functest-docker.yml) | 14 | ||||
| -rw-r--r-- | jjb/functest/xtesting-docker.yml | 221 | ||||
| m--------- | jjb/global-jjb | 0 | ||||
| -rw-r--r-- | jjb/global/slave-params.yml | 1 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.yml | 4 | ||||
| -rw-r--r-- | jjb/xci/xci-merge-jobs.yml | 273 | ||||
| -rwxr-xr-x | jjb/xci/xci-promote.sh | 3 | ||||
| -rwxr-xr-x | jjb/xci/xci-run-functest.sh | 5 | ||||
| -rwxr-xr-x | jjb/xci/xci-set-scenario.sh | 102 | ||||
| -rwxr-xr-x | jjb/xci/xci-start-deployment.sh | 3 | ||||
| -rwxr-xr-x | jjb/xci/xci-start-new-vm.sh | 3 | ||||
| -rw-r--r-- | jjb/xci/xci-verify-jobs.yml | 8 |
20 files changed, 594 insertions, 118 deletions
diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..07b28be4b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "jjb/global-jjb"] + path = jjb/global-jjb + url = https://github.com/lfit/releng-global-jjb diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index b8ae75a1f..123db3e85 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -31,7 +31,9 @@ elif [[ "$DEPLOY_SCENARIO" == *gate* ]]; then fi # Dev or RPM/ISO build -if [[ "$ARTIFACT_VERSION" =~ dev ]]; then +# For upstream deployments we currently only use git repo and not RPM +# Need to decide after Fraser if we want to use RPM or not for upstream +if [[ "$ARTIFACT_VERSION" =~ dev || "$DEPLOY_SCENARIO" =~ "upstream" ]]; then # Settings for deploying from git workspace DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy" NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network" @@ -134,6 +136,11 @@ else DEPLOY_CMD="${DEPLOY_CMD} -i ${INVENTORY_FILE}" fi +if [[ "$DEPLOY_SCENARIO" =~ "upstream" ]]; then + echo "Upstream deployment detected" + DEPLOY_CMD="${DEPLOY_CMD} --upstream" +fi + if [ "$IPV6_FLAG" == "True" ]; then NETWORK_FILE="${NETWORK_SETTINGS_DIR}/network_settings_v6.yaml" elif echo ${DEPLOY_SCENARIO} | grep fdio; then diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh index 68baf59da..c12406cdb 100755 --- a/jjb/apex/apex-download-artifact.sh +++ b/jjb/apex/apex-download-artifact.sh @@ -10,7 +10,10 @@ echo [[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY -if [[ "$ARTIFACT_VERSION" =~ dev ]]; then +# if upstream we do not need to download anything +if [[ "$DEPLOY_SCENARIO" =~ upstream ]]; then + echo "Upstream deployment detected, skipping download artifact" +elif [[ "$ARTIFACT_VERSION" =~ dev ]]; then # dev build GERRIT_PATCHSET_NUMBER=$(echo $GERRIT_REFSPEC | grep -Eo '[0-9]+$') export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}_${GERRIT_PATCHSET_NUMBER}" diff --git a/jjb/container4nfv/yardstick-arm64.yml b/jjb/container4nfv/yardstick-arm64.yml index bd1d8aa85..03c68738a 100644 --- a/jjb/container4nfv/yardstick-arm64.yml +++ b/jjb/container4nfv/yardstick-arm64.yml @@ -5,7 +5,7 @@ - project: name: yardstick-arm64 - project: '{name}' + project: 'yardstick' # ------------------------------- # BRANCH ANCHORS diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index a9a0da2ee..453451f52 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -190,8 +190,10 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'daisy.*-deploy-({pod})?-daily-.*' - 'daisy-kolla-build-.*' + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' block-level: 'NODE' parameters: @@ -333,7 +335,7 @@ - trigger: name: 'daisy-os-nosdn-ovs_dpdk-noha-zte-pod9-daily-master-trigger' triggers: - - timed: '0 10 * * *' + - timed: '0 10,20 * * *' # Basic HA Scenarios - trigger: name: 'daisy-os-nosdn-nofeature-ha-zte-pod9-daily-master-trigger' diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index 006153f6f..c4edd7738 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -215,6 +215,13 @@ enabled: true max-total: 6 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-daily-(build|deploy|test)-(euphrates|master)' + - '{installer}-.*-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + - '(functest|yardstick)-{installer}-(baremetal|virtual|zte-pod9)-daily-(euphrates|master)' + block-level: 'NODE' scm: - git-scm diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index ba76b8abb..8c04ffd53 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -19,7 +19,7 @@ installer: - 'apex' - 'fuel' - # - 'joid' + - 'daisy' arch: - 'x86_64' @@ -29,19 +29,17 @@ - 'sample' - 'congress' - task: - - verify: - auto-trigger-name: 'doctor-verify' - exclude: - installer: 'apex' arch: 'aarch64' + - installer: 'daisy' + arch: 'aarch64' jobs: - 'doctor-verify-{stream}' - 'doctor-verify-unit-test-{stream}' - - 'doctor-{task}-{inspector}-{stream}' - - 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + - 'doctor-verify-{inspector}-{stream}' + - 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' - job-template: name: 'doctor-verify-{stream}' @@ -76,6 +74,7 @@ file-paths: - compare-type: ANT pattern: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -117,16 +116,17 @@ - 'doctor-verify-unit-test-publishers-macro' - job-template: - name: 'doctor-{task}-{inspector}-{stream}' + name: 'doctor-verify-{inspector}-{stream}' disabled: '{obj:disabled}' project-type: 'multijob' scm: - git-scm-gerrit triggers: - - '{auto-trigger-name}': + - 'doctor-verify': project: '{project}' branch: '{branch}' files: 'doctor_tests/**' + builders: - shell: | #!/bin/bash @@ -137,7 +137,7 @@ name: 'doctor-verify-apex-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-apex-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-apex-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -151,7 +151,7 @@ name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -165,7 +165,20 @@ name: 'doctor-verify-fuel-inspector' execution-type: PARALLEL projects: - - name: 'doctor-{task}-fuel-{inspector}-aarch64-{stream}' + - name: 'doctor-verify-fuel-{inspector}-aarch64-{stream}' + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + kill-phase-on: FAILURE + current-parameters: true + git-revision: true + - multijob: + name: 'doctor-verify-daisy-inspector' + execution-type: PARALLEL + projects: + - name: 'doctor-verify-daisy-{inspector}-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -177,7 +190,7 @@ git-revision: true - job-template: - name: 'doctor-{task}-{installer}-{inspector}-{arch}-{stream}' + name: 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' disabled: '{obj:disabled}' node: 'doctor-{installer}-{arch}' wrappers: diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index 541cc786c..432bbbb8b 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -177,9 +177,9 @@ if [[ ${DEPLOY_SCENARIO} =~ ^os-.* ]]; then run_tiers ${tiers} else if [ ${DEPLOY_TYPE} == 'baremetal' ]; then - tiers=(healthcheck smoke patrole features vnf parser) + tiers=(healthcheck smoke features vnf parser) else - tiers=(healthcheck smoke patrole features) + tiers=(healthcheck smoke features) fi run_tiers ${tiers} fi diff --git a/jjb/releng/functest-docker.yml b/jjb/functest/functest-docker.yml index 7c576cf00..909c7b079 100644 --- a/jjb/releng/functest-docker.yml +++ b/jjb/functest/functest-docker.yml @@ -29,7 +29,6 @@ - 'features' - 'components' - 'parser' - - 'patrole' - 'smoke' - 'vnf' - 'restapi' @@ -73,7 +72,7 @@ disabled: '{obj:disabled}' parameters: - - job-parameters: + - functest-job-parameters: project: '{project}' branch: '{branch}' slave_label: 'opnfv-build-ubuntu' @@ -129,10 +128,6 @@ <<: *build-job-settings - name: 'functest-parser-docker-build-arm64-{stream}' <<: *build-job-settings - - name: 'functest-patrole-docker-build-amd64-{stream}' - <<: *build-job-settings - - name: 'functest-patrole-docker-build-arm64-{stream}' - <<: *build-job-settings - name: 'functest-smoke-docker-build-amd64-{stream}' <<: *build-job-settings - name: 'functest-smoke-docker-build-arm64-{stream}' @@ -158,8 +153,6 @@ <<: *manifest-job-settings - name: 'functest-parser-docker-manifest-{stream}' <<: *manifest-job-settings - - name: 'functest-patrole-docker-manifest-{stream}' - <<: *manifest-job-settings - name: 'functest-smoke-docker-manifest-{stream}' <<: *manifest-job-settings - name: 'functest-vnf-docker-manifest-{stream}' @@ -175,7 +168,7 @@ name: 'functest-{image}-docker-build-{arch_tag}-{stream}' disabled: '{obj:disabled}' parameters: - - job-parameters: + - functest-job-parameters: project: '{project}' branch: '{branch}' slave_label: '{slave_label}' @@ -237,7 +230,7 @@ # parameter macro - parameter: - name: job-parameters + name: functest-job-parameters parameters: - project-parameter: project: '{project}' @@ -297,3 +290,4 @@ jalausuch@suse.com morgan.richomme@orange.com cedric.ollivier@orange.com feng.xiaowei@zte.com.cn juha.kosonen@nokia.com wangwulin@huawei.com + valentin.boucher@kontron.com diff --git a/jjb/functest/xtesting-docker.yml b/jjb/functest/xtesting-docker.yml new file mode 100644 index 000000000..dc26344e3 --- /dev/null +++ b/jjb/functest/xtesting-docker.yml @@ -0,0 +1,221 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: xtesting-docker + + project: functest-xtesting + + stream: + - master: + branch: '{stream}' + disabled: false + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + - 'arm64': + slave_label: 'opnfv-build-ubuntu-arm' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + GERRIT_REFNAME=$GERRIT_REFNAME + DOCKERFILE=$DOCKERFILE + kill-phase-on: FAILURE + abort-all-jobs: false + + manifest-job-settings: &manifest-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: + GERRIT_REFNAME=$GERRIT_REFNAME + kill-phase-on: FAILURE + abort-all-jobs: false + + # yamllint enable rule:key-duplicates + jobs: + - "xtesting-docker-{stream}" + - "xtesting-docker-build-{arch_tag}-{stream}" + - "xtesting-docker-manifest-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'xtesting-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - xtesting-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/30 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' + + builders: + - multijob: + name: 'build xtesting images' + execution-type: PARALLEL + projects: + - name: 'xtesting-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'xtesting-docker-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish xtesting manifests' + execution-type: PARALLEL + projects: + - name: 'xtesting-docker-manifest-{stream}' + <<: *manifest-job-settings + + publishers: + - 'xtesting-amd64-recipients' + - 'xtesting-arm64-recipients' + +- job-template: + name: 'xtesting-docker-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - xtesting-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + scm: + - git-scm + builders: + - shell: | + #!/bin/bash -ex + sudo arch={arch_tag} bash ./build.sh + exit $? + +- job-template: + name: 'xtesting-docker-manifest-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'opnfv-build-ubuntu' + description: 'Slave label on Jenkins' + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + + + disabled: '{obj:disabled}' + + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template $REPO/xtesting:ARCH-$tag \ + --target $REPO/xtesting:$tag + exit $? + +- parameter: + name: xtesting-job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: GERRIT_REFNAME + default: "" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" + - string: + name: DOCKERFILE + default: "Dockerfile" + description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "{arch_tag}" + description: "If set, this value will be added to the docker image tag as a prefix" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest images" + +# publisher macros +- publisher: + name: 'xtesting-arm64-recipients' + publishers: + - email: + recipients: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + delia.popescu@enea.com + +- publisher: + name: 'xtesting-amd64-recipients' + publishers: + - email: + recipients: > + jalausuch@suse.com morgan.richomme@orange.com + cedric.ollivier@orange.com feng.xiaowei@zte.com.cn + juha.kosonen@nokia.com wangwulin@huawei.com + valentin.boucher@kontron.com diff --git a/jjb/global-jjb b/jjb/global-jjb new file mode 160000 +Subproject 779110b5cd63f3eabb63598a1be79d9b9ba8546 diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 86b369bcd..6171a6abd 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -983,6 +983,7 @@ - 'doctor-fuel-x86_64' - 'doctor-fuel-aarch64' - 'doctor-joid-x86_64' + - 'doctor-daisy-x86_64' - 'not-specified' default-slaves: - 'not-specified' diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index ac0869d9c..55cbc5f1d 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -181,10 +181,6 @@ project: 'yardstick' <<: *euphrates <<: *other-receivers - - 'xtesting': - project: 'functest-xtesting' - <<: *master - <<: *other-receivers # projects with jobs for danube - 'dovetail': project: 'dovetail' diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml index 92eea190f..8a3a7cfbc 100644 --- a/jjb/xci/xci-merge-jobs.yml +++ b/jjb/xci/xci-merge-jobs.yml @@ -2,6 +2,7 @@ - project: name: 'opnfv-xci-merge' + project: releng-xci # ------------------------------- # branches # ------------------------------- @@ -19,34 +20,35 @@ - opensuse: disabled: false # ------------------------------- + # scenarios + # ------------------------------- + scenario: + - 'os-odl-sfc': + project: sfc + # ------------------------------- # type # ------------------------------- type: - virtual # ------------------------------- - # patch verification phases + # postmerge promotion phases # ------------------------------- phase: - 'deploy' - 'healthcheck' - - 'promote' - # ------------------------------- - # scenarios - # ------------------------------- - scenario: - - 'os-odl-sfc': - project: sfc # ------------------------------- # jobs # ------------------------------- jobs: - - 'xci-{scenario}-{type}-{distro}-merge-{stream}' - - 'xci-{phase}-{type}-{distro}-merge-{stream}' + - 'xci-{scenario}-{type}-merge-{stream}' + - 'xci-{scenario}-{type}-promote-merge-{stream}' + - 'xci-merge-{distro}-{type}-{stream}' + - 'xci-merge-{distro}-{phase}-{type}-{stream}' # ------------------------------- # job templates # ------------------------------- - job-template: - name: 'xci-{scenario}-{type}-{distro}-merge-{stream}' + name: 'xci-{scenario}-{type}-merge-{stream}' project-type: multijob @@ -56,22 +58,6 @@ properties: - logrotate-default - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'xci-verify-{distro}-.*' - - 'xci-.*-{distro}-merge-.*' - - 'openstack-bifrost-verify-{distro}-.*' - - 'xci-osa-verify-{distro}-.*' - - 'xci-osa-periodic-{distro}-.*' - - 'xci-(os|k8s).*?-virtual-{distro}-.*' - block-level: 'NODE' - - throttle: - max-per-node: 2 - max-total: 10 - categories: - - xci-verify-virtual - option: category wrappers: - ssh-agent-wrapper @@ -80,12 +66,7 @@ - fix-workspace-permissions scm: - - git: - credentials-id: '$SSH_CREDENTIAL_ID' - url: 'https://gerrit.opnfv.org/gerrit/releng-xci' - branches: - - 'origin/master' - timeout: 15 + - git-scm-gerrit triggers: - gerrit: @@ -109,14 +90,14 @@ parameters: - project-parameter: - project: '{obj:project}' + project: $GERRIT_PROJECT branch: '{branch}' - label: name: SLAVE_LABEL default: 'xci-virtual' - string: - name: DEPLOY_SCENARIO - default: '{scenario}' + name: DISTRO + default: 'all' - string: name: CLEAN_DIB_IMAGES default: 'true' @@ -127,18 +108,139 @@ builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Node: $NODE_NAME" + - 'xci-merge-set-scenario-macro' + - multijob: + name: deploy and test + condition: SUCCESSFUL + projects: + - name: 'xci-merge-opensuse-{type}-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=opensuse + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - name: 'xci-merge-ubuntu-{type}-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=ubuntu + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - name: 'xci-merge-centos-{type}-{stream}' + current-parameters: false + predefined-parameters: | + DISTRO=centos + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - multijob: + name: promote + condition: SUCCESSFUL + projects: + - name: 'xci-{scenario}-{type}-promote-merge-{stream}' + current-parameters: true + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + +- job-template: + name: 'xci-merge-{distro}-{type}-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-verify-{distro}-.*' + - 'xci-merge-{distro}-.*' + - 'openstack-bifrost-verify-{distro}-.*' + - 'xci-osa-verify-{distro}-.*' + - 'xci-osa-periodic-{distro}-.*' + block-level: 'NODE' + - throttle: + max-per-node: 1 + max-total: 3 + categories: + - xci-verify-virtual + option: category + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + scm: + - git-scm-gerrit + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'xci-virtual' + - string: + name: DISTRO + default: '{distro}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + builders: + - 'xci-merge-set-scenario-macro' + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - description-setter: + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - multijob: name: deploy condition: SUCCESSFUL projects: - - name: 'xci-deploy-{type}-{distro}-merge-{stream}' + - name: 'xci-merge-{distro}-deploy-{type}-{stream}' current-parameters: true predefined-parameters: | - PROJECT=$PROJECT DISTRO={distro} - DEPLOY_SCENARIO={scenario} CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -150,19 +252,18 @@ name: healthcheck condition: SUCCESSFUL projects: - - name: 'xci-healthcheck-{type}-{distro}-merge-{stream}' + - name: 'xci-merge-{distro}-healthcheck-{type}-{stream}' current-parameters: true predefined-parameters: | - PROJECT=$PROJECT DISTRO={distro} - DEPLOY_SCENARIO={scenario} CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES - FUNCTEST_MODE=tier - FUNCTEST_TIER=healthcheck + GERRIT_PROJECT=$GERRIT_PROJECT GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + FUNCTEST_MODE=tier + FUNCTEST_TIER=healthcheck node-parameters: true kill-phase-on: NEVER abort-all-job: true @@ -176,7 +277,7 @@ !include-raw: ./xci-cleanup.sh - job-template: - name: 'xci-{phase}-{type}-{distro}-merge-{stream}' + name: 'xci-merge-{distro}-{phase}-{type}-{stream}' disabled: false @@ -196,14 +297,11 @@ parameters: - string: name: PROJECT - default: 'releng-xci' + default: $GERRIT_PROJECT - string: name: DISTRO default: 'ubuntu' - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-noha' - - string: name: FUNCTEST_MODE default: 'tier' - string: @@ -219,51 +317,90 @@ name: OPNFV_RELENG_DEV_PATH default: $WORKSPACE/ - string: - name: INSTALLER_TYPE - default: 'osa' - - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" - ssh-agent-wrapper - build-timeout: timeout: 240 - fix-workspace-permissions scm: - - git: - credentials-id: '$SSH_CREDENTIAL_ID' - url: 'https://gerrit.opnfv.org/gerrit/releng-xci' - branches: - - 'origin/master' - timeout: 15 + - git-scm-gerrit builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - 'xci-merge-{phase}-macro' +- job-template: + name: 'xci-{scenario}-{type}-promote-merge-{stream}' + + disabled: false + + concurrent: false + + properties: + - logrotate-default + + parameters: + - string: + name: PROJECT + default: $GERRIT_PROJECT + - string: + name: DISTRO + default: 'all' + - string: + name: OPNFV_RELENG_DEV_PATH + default: $WORKSPACE/ + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + + wrappers: + - inject: + properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties" + - ssh-agent-wrapper + - build-timeout: + timeout: 240 + - fix-workspace-permissions + + builders: + - description-setter: + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" + - 'xci-merge-promote-macro' + # ------------------------------- # builder macros # ------------------------------- - builder: - name: 'xci-merge-deploy-macro' + name: 'xci-merge-set-scenario-macro' builders: - shell: - !include-raw: ./xci-start-new-vm.sh - - shell: - !include-raw: ./xci-start-deployment.sh + !include-raw: ./xci-set-scenario.sh + +- builder: + name: 'xci-merge-deploy-macro' + builders: + - shell: | + #!/bin/bash + echo "Hello World" - builder: name: 'xci-merge-healthcheck-macro' builders: - - shell: - !include-raw: ./xci-run-functest.sh + - shell: | + #!/bin/bash + echo "Hello World" - builder: name: 'xci-merge-promote-macro' builders: - - shell: - !include-raw: ./xci-promote.sh + - shell: | + #!/bin/bash + echo "Hello World" diff --git a/jjb/xci/xci-promote.sh b/jjb/xci/xci-promote.sh index 98ad7ff4a..62818abee 100755 --- a/jjb/xci/xci-promote.sh +++ b/jjb/xci/xci-promote.sh @@ -14,6 +14,9 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the healthcheck if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the healthcheck!" diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh index 1bc569aef..1f616de1f 100755 --- a/jjb/xci/xci-run-functest.sh +++ b/jjb/xci/xci-run-functest.sh @@ -17,6 +17,9 @@ set -o pipefail # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the healthcheck if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the healthcheck!" @@ -48,7 +51,7 @@ if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENT exit 0 fi -ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "cd releng-xci/xci && PATH=/home/devuser/.local/bin:$PATH ansible-playbook -i installer/osa/files/$XCI_FLAVOR/inventory playbooks/prepare-functest.yml" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "cd /root/releng-xci/xci/playbooks && ansible-playbook -i inventory prepare-functest.yml" echo "Running functest" ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "/root/run-functest.sh" echo "Functest log" diff --git a/jjb/xci/xci-set-scenario.sh b/jjb/xci/xci-set-scenario.sh index 79b31bfb0..c602957a9 100755 --- a/jjb/xci/xci-set-scenario.sh +++ b/jjb/xci/xci-set-scenario.sh @@ -16,15 +16,76 @@ set -o pipefail # what you are doing. #---------------------------------------------------------------------- +# This function allows developers to specify the impacted scenario by adding +# the info about installer and scenario into the commit message or using +# the topic branch names. This results in either skipping the real verification +# totally or skipping the determining the installer and scenario programmatically. +# It is important to note that this feature is only available to generic scenarios +# and only single installer/scenario pair is allowed. +# The input in commit message should be placed at the end of the commit message body, +# before the signed-off and change-id lines. +# +# Pattern to be searched in Commit Message +# deploy-scenario:<scenario-name> +# installer-type:<installer-type> +# Examples: +# deploy-scenario:os-odl-nofeature +# installer-type:osa +# +# deploy-scenario:k8-nosdn-nofeature +# installer-type:kubespray +# +# Patterns to be searched in topic branch name +# skip-verify +# skip-deployment +# force-verify +function override_generic_scenario() { + echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" + + # process topic branch names + if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then + # skip the real verification + echo "Skipping verify!" + echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + exit 0 + elif [[ "$GERRIT_TOPIC" =~ 'force-verify' ]]; then + # Run the deployment with default installer and scenario when multiple things change + # and we want to force that. + echo "Recording the installer 'osa' and scenario 'os-nosdn-nofeature' for downstream jobs" + echo "Forcing CI verification of default scenario and installer!" + echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties + exit 0 + fi + + # process commit message + if [[ "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "installer-type:" && "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "deploy-scenario:" ]]; then + INSTALLER_TYPE=$(echo $GERRIT_CHANGE_COMMIT_MESSAGE | awk '/installer-type:/' RS=" " | cut -d":" -f2) + DEPLOY_SCENARIO=$(echo $GERRIT_CHANGE_COMMIT_MESSAGE | awk '/deploy-scenario:/' RS=" " | cut -d":" -f2) + + if [[ -z "$INSTALLER_TYPE" || -z "$DEPLOY_SCENARIO" ]]; then + echo "Installer type or deploy scenario is not specified. Falling back to programmatically determining them." + else + echo "Recording the installer '$INSTALLER_TYPE' and scenario '$DEPLOY_SCENARIO' for downstream jobs" + echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties + echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties + exit 0 + fi + else + echo "Installer type or deploy scenario is not specified. Falling back to programmatically determining them." + fi +} + # This function determines the impacted generic scenario by processing the # change and using diff to see what changed. If changed files belong to a scenario # its name gets recorded for deploying and testing the right scenario. # -# Pattern -# releng-xci/scenarios/<scenario>/<impacted files>: <scenario> -# releng-xci/xci/installer/osa/<impacted files>: os-nosdn-nofeature -# releng-xci/xci/installer/kubespray/<impacted files>: k8-nosdn-nofeature -# the rest: os-nosdn-nofeature +# Pattern to be searched in Changeset +# releng-xci/scenarios/<scenario>/<impacted files>: <scenario> +# releng-xci/xci/installer/osa/<impacted files>: os-nosdn-nofeature +# releng-xci/xci/installer/kubespray/<impacted files>: k8-nosdn-nofeature +# the rest: os-nosdn-nofeature function determine_generic_scenario() { echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" @@ -47,6 +108,12 @@ function determine_generic_scenario() { ;; esac done + + # extract releng-xci sha + RELENG_XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha which is same as releng-xci sha for generic scenarios + SCENARIO_SHA=$RELENG_XCI_SHA } # This function determines the impacted external scenario by processing the Gerrit @@ -54,7 +121,7 @@ function determine_generic_scenario() { # its name gets recorded for deploying and testing the right scenario. # # Pattern -# <project-repo>/scenarios/<scenario>/<impacted files>: <scenario> +# <project-repo>/scenarios/<scenario>/<impacted files>: <scenario> function determine_external_scenario() { echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC" @@ -73,6 +140,12 @@ function determine_external_scenario() { # process the diff to find out what scenario(s) are impacted - there should only be 1 DEPLOY_SCENARIO+=$(git diff HEAD^..HEAD --name-only | grep scenarios | awk -F '[/|/]' '{print $2}' | uniq) + + # extract releng-xci sha + RELENG_XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD) + + # extract scenario sha + SCENARIO_SHA=$(cd $WORK_DIRECTORY/$GERRIT_PROJECT && git rev-parse HEAD) } echo "Determining the impacted scenario" @@ -87,14 +160,8 @@ GERRIT_TOPIC="${GERRIT_TOPIC:-''}" WORK_DIRECTORY=/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO /bin/rm -rf $WORK_DIRECTORY && mkdir -p $WORK_DIRECTORY -# skip the healthcheck if the patch doesn't impact the deployment -if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then - echo "Skipping verify!" - echo "DEPLOY_SCENARIO=os-nosdn-nofeature" > $WORK_DIRECTORY/scenario.properties - exit 0 -fi - if [[ $GERRIT_PROJECT == "releng-xci" ]]; then + override_generic_scenario determine_generic_scenario else determine_external_scenario @@ -124,9 +191,16 @@ esac # save the installer and scenario names into java properties file # so they can be injected to downstream jobs via envInject -echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' for downstream jobs" +echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' and SHAs for downstream jobs" echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties +echo "RELENG_XCI_SHA=$RELENG_XCI_SHA" >> $WORK_DIRECTORY/scenario.properties +echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties + +# skip scenario support check if the job is promotion job +if [[ "$JOB_NAME" =~ (os|k8) ]]; then + exit 0 +fi # skip the deployment if the scenario is not supported on this distro OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml diff --git a/jjb/xci/xci-start-deployment.sh b/jjb/xci/xci-start-deployment.sh index 0628829ea..102ca41c3 100755 --- a/jjb/xci/xci-start-deployment.sh +++ b/jjb/xci/xci-start-deployment.sh @@ -14,6 +14,9 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then echo "Skipping the deployment!" diff --git a/jjb/xci/xci-start-new-vm.sh b/jjb/xci/xci-start-new-vm.sh index c2f1f991f..7874e6776 100755 --- a/jjb/xci/xci-start-new-vm.sh +++ b/jjb/xci/xci-start-new-vm.sh @@ -14,6 +14,9 @@ # what you are doing. #---------------------------------------------------------------------- +# ensure GERRIT_TOPIC is set +GERRIT_TOPIC="${GERRIT_TOPIC:-''}" + # skip the deployment if the patch doesn't impact the deployment if [[ "$GERRIT_TOPIC" =~ 'skip-verify' ]]; then echo "Skipping the deployment!" diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 4c9ff9e69..d78dc82d9 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -66,7 +66,7 @@ use-build-blocker: true blocking-jobs: - 'xci-verify-{distro}-.*' - - 'xci-.*-{distro}-merge-.*' + - 'xci-merge-{distro}-.*' - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' @@ -110,6 +110,12 @@ forbidden-file-paths: - compare-type: ANT pattern: 'xci/scripts/vm/**' + - compare-type: ANT + pattern: 'docs/**' + - compare-type: ANT + pattern: 'prototypes/**' + - compare-type: ANT + pattern: 'upstream/**' - project-compare-type: 'REG_EXP' project-pattern: 'sfc|sdnvpn' branches: |