diff options
Diffstat (limited to 'jjb')
52 files changed, 920 insertions, 347 deletions
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index 74f67ce5f..3a2ca606b 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail -APEX_PKGS="common undercloud" # removed onos for danube +APEX_PKGS="common undercloud onos" IPV6_FLAG=False # log info to console diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh index 206c627ec..52c3c67ec 100755 --- a/jjb/apex/apex-download-artifact.sh +++ b/jjb/apex/apex-download-artifact.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail -APEX_PKGS="common undercloud" # removed onos for danube +APEX_PKGS="common undercloud onos" # log info to console echo "Downloading the Apex artifact. This could take some time..." diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index 25870bb35..f53451d41 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -126,13 +126,13 @@ elif [ "$ARTIFACT_TYPE" == 'rpm' ]; then RPM_INSTALL_PATH=$BUILD_DIRECTORY/noarch RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL) VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//') - for pkg in common undercloud; do # removed onos for danube + for pkg in common undercloud onos; do RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}" done SRPM_INSTALL_PATH=$BUILD_DIRECTORY SRPM_LIST=$SRPM_INSTALL_PATH/$(basename $OPNFV_SRPM_URL) VERSION_EXTENSION=$(echo $(basename $OPNFV_SRPM_URL) | sed 's/opnfv-apex-//') - for pkg in common undercloud; do # removed onos for danube + for pkg in common undercloud onos; do SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}" done diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 50502d77a..a395cf2e5 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -204,23 +204,23 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true -# - multijob: -# name: functest-smoke -# condition: SUCCESSFUL -# projects: -# - name: 'functest-apex-virtual-suite-{stream}' -# current-parameters: false -# predefined-parameters: | -# DEPLOY_SCENARIO={verify-scenario} -# FUNCTEST_SUITE_NAME=healthcheck -# GERRIT_BRANCH=$GERRIT_BRANCH -# GERRIT_REFSPEC=$GERRIT_REFSPEC -# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER -# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE -# node-parameters: true -# kill-phase-on: FAILURE -# abort-all-job: true -# git-revision: false + - multijob: + name: functest-smoke + condition: SUCCESSFUL + projects: + - name: 'functest-apex-virtual-suite-{stream}' + current-parameters: false + predefined-parameters: | + DEPLOY_SCENARIO={verify-scenario} + FUNCTEST_SUITE_NAME=healthcheck + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false # Verify Scenario Gate - job-template: @@ -294,27 +294,28 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + GERRIT_EVENT_COMMENT_TEXT=$GERRIT_EVENT_COMMENT_TEXT node-parameters: true kill-phase-on: FAILURE abort-all-job: true git-revision: true -# - multijob: -# name: functest-smoke -# condition: SUCCESSFUL -# projects: -# - name: 'functest-apex-virtual-suite-{stream}' -# current-parameters: false -# predefined-parameters: | -# DEPLOY_SCENARIO={verify-scenario} -# FUNCTEST_SUITE_NAME=healthcheck -# GERRIT_BRANCH=$GERRIT_BRANCH -# GERRIT_REFSPEC=$GERRIT_REFSPEC -# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER -# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE -# node-parameters: true -# kill-phase-on: FAILURE -# abort-all-job: true -# git-revision: false + - multijob: + name: functest-smoke + condition: SUCCESSFUL + projects: + - name: 'functest-apex-virtual-suite-{stream}' + current-parameters: false + predefined-parameters: | + DEPLOY_SCENARIO={verify-scenario} + FUNCTEST_SUITE_NAME=healthcheck + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - job-template: name: 'apex-runner-cperf-{stream}' diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index 84a899f57..752cf281d 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -204,23 +204,23 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true -# - multijob: -# name: functest-smoke -# condition: SUCCESSFUL -# projects: -# - name: 'functest-apex-virtual-suite-{stream}' -# current-parameters: false -# predefined-parameters: | -# DEPLOY_SCENARIO={verify-scenario} -# FUNCTEST_SUITE_NAME=healthcheck -# GERRIT_BRANCH=$GERRIT_BRANCH -# GERRIT_REFSPEC=$GERRIT_REFSPEC -# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER -# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE -# node-parameters: true -# kill-phase-on: FAILURE -# abort-all-job: true -# git-revision: false + - multijob: + name: functest-smoke + condition: SUCCESSFUL + projects: + - name: 'functest-apex-virtual-suite-{stream}' + current-parameters: false + predefined-parameters: | + DEPLOY_SCENARIO={verify-scenario} + FUNCTEST_SUITE_NAME=healthcheck + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false # Verify Scenario Gate - job-template: @@ -294,27 +294,28 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + GERRIT_EVENT_COMMENT_TEXT=$GERRIT_EVENT_COMMENT_TEXT node-parameters: true kill-phase-on: FAILURE abort-all-job: true git-revision: true -# - multijob: -# name: functest-smoke -# condition: SUCCESSFUL -# projects: -# - name: 'functest-apex-virtual-suite-{stream}' -# current-parameters: false -# predefined-parameters: | -# DEPLOY_SCENARIO={verify-scenario} -# FUNCTEST_SUITE_NAME=healthcheck -# GERRIT_BRANCH=$GERRIT_BRANCH -# GERRIT_REFSPEC=$GERRIT_REFSPEC -# GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER -# GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE -# node-parameters: true -# kill-phase-on: FAILURE -# abort-all-job: true -# git-revision: false + - multijob: + name: functest-smoke + condition: SUCCESSFUL + projects: + - name: 'functest-apex-virtual-suite-{stream}' + current-parameters: false + predefined-parameters: | + DEPLOY_SCENARIO={verify-scenario} + FUNCTEST_SUITE_NAME=healthcheck + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - job-template: name: 'apex-runner-cperf-{stream}' diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 1531ff144..55d8ff975 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -283,31 +283,31 @@ - trigger: name: 'fuel-os-odl_l2-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 0 * * 1' + - timed: '' - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 0 * * 2' + - timed: '' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 0 * * 3' + - timed: '' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 0 * * 4' + - timed: '' - trigger: name: 'fuel-os-odl_l2-nofeature-noha-armband-baremetal-master-trigger' triggers: - - timed: '0 0 * * 5' + - timed: '' - trigger: name: 'fuel-os-odl_l2-sfc-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 0,20 * * 6' + - timed: '' - trigger: name: 'fuel-os-odl_l2-sfc-noha-armband-baremetal-master-trigger' triggers: - - timed: '0 0,20 * * 7' + - timed: '' #---------------------------------------------------------------------- # Enea Armband CI Baremetal Triggers running against danube branch @@ -315,31 +315,31 @@ - trigger: name: 'fuel-os-odl_l2-nofeature-ha-armband-baremetal-danube-trigger' triggers: - - timed: '0 4 * * 1,2,3,4,5' + - timed: '0 0,16 * * 2,4' - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-danube-trigger' triggers: - - timed: '0 8 * * 1,2,3,4,5' + - timed: '0 0 * * 1,5,7' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-armband-baremetal-danube-trigger' triggers: - - timed: '0 12 * * 1,2,3,4,5' + - timed: '0 16 * * 1,5,7' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-armband-baremetal-danube-trigger' triggers: - - timed: '0 16 * * 1,2,3,4,5' + - timed: '0 8 * * 2,4,6' - trigger: name: 'fuel-os-odl_l2-nofeature-noha-armband-baremetal-danube-trigger' triggers: - - timed: '0 20 * * 1,2,3,4,5' + - timed: '0 8 * * 1,3,5,7' - trigger: name: 'fuel-os-odl_l2-sfc-ha-armband-baremetal-danube-trigger' triggers: - - timed: '0 4,8 * * 6,7' + - timed: '0 0 * * 3,6' - trigger: name: 'fuel-os-odl_l2-sfc-noha-armband-baremetal-danube-trigger' triggers: - - timed: '0 12,16 * * 6,7' + - timed: '0 16 * * 3,6' #--------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against master branch #--------------------------------------------------------------- diff --git a/jjb/barometer/barometer-build.sh b/jjb/barometer/barometer-build.sh new file mode 100644 index 000000000..e40841bc2 --- /dev/null +++ b/jjb/barometer/barometer-build.sh @@ -0,0 +1,21 @@ +set -x + +OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +OPNFV_ARTIFACT_URL="$GS_URL/$OPNFV_ARTIFACT_VERSION/" + +# log info to console +echo "Starting the build of Barometer RPMs" +echo "------------------------------------" +echo + +cd ci +./install_dependencies.sh +./build_rpm.sh +cd $WORKSPACE + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_ARTIFACT_URL=$OPNFV_ARTIFACT_URL" +) > $WORKSPACE/opnfv.properties + diff --git a/jjb/barometer/barometer-upload-artifact.sh b/jjb/barometer/barometer-upload-artifact.sh new file mode 100644 index 000000000..817cc5770 --- /dev/null +++ b/jjb/barometer/barometer-upload-artifact.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -o nounset +set -o pipefail + +RPM_WORKDIR=$WORKSPACE/rpmbuild +RPM_DIR=$RPM_WORKDIR/RPMS/x86_64/ +cd $WORKSPACE/ + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +# upload property files +gsutil cp $WORKSPACE/opnfv.properties gs://$OPNFV_ARTIFACT_URL/opnfv.properties > gsutil.properties.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 + +echo "Uploading the barometer RPMs to artifacts.opnfv.org" +echo "---------------------------------------------------" +echo + +gsutil -m cp -r $RPM_DIR/* gs://$OPNFV_ARTIFACT_URL > $WORKSPACE/gsutil.log 2>&1 + +# Check if the RPMs were pushed +gsutil ls gs://$OPNFV_ARTIFACT_URL > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading barometer RPMs to gs://$OPNFV_ARTIFACT_URL!" + echo "Check log $WORKSPACE/gsutil.log on the appropriate build server" + exit 1 +fi + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$OPNFV_ARTIFACT_URL/*.rpm > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$OPNFV_ARTIFACT_URL/opnfv.properties > /dev/null 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available at $OPNFV_ARTIFACT_URL" + +#cleanup the RPM repo from the build machine. +rm -rf $RPM_WORKDIR diff --git a/jjb/barometer/barometer.yml b/jjb/barometer/barometer.yml index 68b8a04c0..2d3e972f8 100644 --- a/jjb/barometer/barometer.yml +++ b/jjb/barometer/barometer.yml @@ -115,7 +115,6 @@ - shell: | pwd cd src - ./install_build_deps.sh make clobber make @@ -135,6 +134,8 @@ - project-parameter: project: '{project}' branch: '{branch}' + - barometer-project-parameter: + gs-pathname: '{gs-pathname}' - 'opnfv-build-centos-defaults' scm: @@ -144,8 +145,18 @@ - timed: '@midnight' builders: - - shell: | - pwd - cd ci - ./install_dependencies.sh - ./build_rpm.sh + - shell: + !include-raw-escape: ./barometer-build.sh + - shell: + !include-raw-escape: ./barometer-upload-artifact.sh + +######################## +# parameter macros +######################## +- parameter: + name: barometer-project-parameter + parameters: + - string: + name: GS_URL + default: '$GS_BASE{gs-pathname}' + description: "URL to Google Storage." diff --git a/jjb/ci_gate_security/anteater-clone-all-repos.sh b/jjb/ci_gate_security/anteater-clone-all-repos.sh new file mode 100755 index 000000000..8a9e73d85 --- /dev/null +++ b/jjb/ci_gate_security/anteater-clone-all-repos.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +set -o errexit +set -o pipefail +set -o nounset +export PATH=$PATH:/usr/local/bin/ + + +#WORKSPACE="$(pwd)" + +cd $WORKSPACE +if [ ! -d "$WORKSPACE/allrepos" ]; then + mkdir $WORKSPACE/allrepos +fi + +cd $WORKSPACE/allrepos + +declare -a PROJECT_LIST +EXCLUDE_PROJECTS="All-Projects|All-Users|securedlab" + +PROJECT_LIST=($(ssh gerrit.opnfv.org -p 29418 gerrit ls-projects | egrep -v $EXCLUDE_PROJECTS)) +echo "PROJECT_LIST=(${PROJECT_LIST[*]})" > $WORKSPACE/opnfv-projects.sh + +for PROJECT in ${PROJECT_LIST[@]}; do + echo "> Cloning $PROJECT" + if [ ! -d "$PROJECT" ]; then + git clone "https://gerrit.opnfv.org/gerrit/$PROJECT.git" + else + pushd "$PROJECT" > /dev/null + git pull -f + popd > /dev/null + fi +done diff --git a/jjb/ci_gate_security/anteater-report-to-gerrit.sh b/jjb/ci_gate_security/anteater-report-to-gerrit.sh new file mode 100644 index 000000000..fc3018fb4 --- /dev/null +++ b/jjb/ci_gate_security/anteater-report-to-gerrit.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +set -o pipefail +export PATH=$PATH:/usr/local/bin/ +EXITSTATUS=0 + +# This Log should always exist +if [[ -e securityaudit.log ]] ; then + echo -e "\nposting security audit report to gerrit...\n" + + #check if log has errors + if grep ERROR securityaudit.log; then + EXITSTATUS=1 + fi + + cat securityaudit.log | awk -F"ERROR - " '{print $2}' > shortlog + + ssh -p 29418 gerrit.opnfv.org \ + "gerrit review -p $GERRIT_PROJECT \ + -m \"$(cat shortlog)\" \ + $GERRIT_PATCHSET_REVISION \ + --notify NONE" + + exit $EXITSTATUS +fi diff --git a/jjb/ci_gate_security/anteater-security-audit-weekly.sh b/jjb/ci_gate_security/anteater-security-audit-weekly.sh new file mode 100644 index 000000000..436a173bc --- /dev/null +++ b/jjb/ci_gate_security/anteater-security-audit-weekly.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 + +echo "--------------------------------------------------------" +vols="-v $WORKSPACE/allrepos/:/home/opnfv/anteater/allrepos/" +echo "Pulling releng-anteater docker image" +echo "--------------------------------------------------------" +docker pull opnfv/releng-anteater +echo "--------------------------------------------------------" +cmd="docker run -id $vols opnfv/releng-anteater /bin/bash" +echo "Running docker command $cmd" +container_id=$($cmd) +echo "Container ID is $container_id" +source $WORKSPACE/opnfv-projects.sh +for project in "${PROJECT_LIST[@]}" + +do + cmd="anteater --project testproj --path /home/opnfv/anteater/allrepos/$project" + echo "Executing command inside container" + echo "$cmd" + echo "--------------------------------------------------------" + docker exec $container_id $cmd > $WORKSPACE/"$project".securityaudit.log 2>&1 +done + +exit_code=$? +echo "--------------------------------------------------------" +echo "Stopping docker container with ID $container_id" +docker stop $container_id + + +#gsutil cp $WORKSPACE/securityaudit.log \ +# gs://$GS_URL/$PROJECT-securityaudit-weekly.log 2>&1 +# +#gsutil -m setmeta \ +# -h "Content-Type:text/html" \ +# -h "Cache-Control:private, max-age=0, no-transform" \ +# gs://$GS_URL/$PROJECT-securityaudit-weekly.log > /dev/null 2>&1 diff --git a/jjb/ci_gate_security/anteater-security-audit.sh b/jjb/ci_gate_security/anteater-security-audit.sh new file mode 100644 index 000000000..9bd3cc34f --- /dev/null +++ b/jjb/ci_gate_security/anteater-security-audit.sh @@ -0,0 +1,28 @@ +#!/bin/bash +cd $WORKSPACE +echo "Generating patchset file to list changed files" +git diff HEAD^1 --name-only | sed "s#^#/home/opnfv/anteater/$PROJECT/#" > $WORKSPACE/patchset +echo "Changed files are" +echo "--------------------------------------------------------" +cat $WORKSPACE/patchset +echo "--------------------------------------------------------" + +vols="-v $WORKSPACE:/home/opnfv/anteater/$PROJECT" +envs="-e PROJECT=$PROJECT" + +echo "Pulling releng-anteater docker image" +echo "--------------------------------------------------------" +docker pull opnfv/releng-anteater +echo "--------------------------------------------------------" + +cmd="docker run -i $envs $vols --rm opnfv/releng-anteater \ +/home/opnfv/venv/bin/anteater --project $PROJECT --patchset /home/opnfv/anteater/$PROJECT/patchset" +echo "Running docker container" +echo "$cmd" +$cmd > $WORKSPACE/securityaudit.log 2>&1 +exit_code=$? +echo "--------------------------------------------------------" +echo "Docker container exited with code: $exit_code" +echo "--------------------------------------------------------" +cat securityaudit.log +exit 0 diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml index 732df8925..e2f6ceb7b 100644 --- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml +++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml @@ -1,5 +1,6 @@ +# SPDX-license-identifier: Apache-2.0 ######################## -# Job configuration for opnfv-lint +# Job configuration for opnfv-anteater (security audit) ######################## - project: @@ -9,6 +10,7 @@ jobs: - 'opnfv-security-audit-verify-{stream}' + - 'opnfv-security-audit-weekly-{stream}' stream: - master: @@ -20,14 +22,42 @@ # job templates ######################## - job-template: + name: 'opnfv-security-audit-weekly-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-build3' + description: 'Slave label on Jenkins' + - project-parameter: + project: releng + branch: '{branch}' + + triggers: + - timed: '@weekly' + + builders: + - anteater-security-audit-weekly + +- job-template: name: 'opnfv-security-audit-verify-{stream}' disabled: '{obj:disabled}' parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-build3' + description: 'Slave label on Jenkins' - project-parameter: project: $GERRIT_PROJECT branch: '{branch}' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." scm: - git-scm-gerrit @@ -47,59 +77,42 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'sandbox' + project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|functest|octopus|pharos|releng|sandbox' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' file-paths: - compare-type: ANT - pattern: '**/*.py' - skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true + pattern: '**' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true builders: - - security-audit-python-code + - anteater-security-audit - report-security-audit-result-to-gerrit ######################## # builder macros ######################## - builder: - name: security-audit-python-code + name: anteater-security-audit builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # this is where the security/license audit script will be executed - echo "Hello World!" + - shell: + !include-raw: ./anteater-security-audit.sh + - builder: name: report-security-audit-result-to-gerrit builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # If no violations were found, no lint log will exist. - if [[ -e securityaudit.log ]] ; then - echo -e "\nposting security audit report to gerrit...\n" - - cat securityaudit.log - echo - - ssh -p 29418 gerrit.opnfv.org \ - "gerrit review -p $GERRIT_PROJECT \ - -m \"$(cat securityaudit.log)\" \ - $GERRIT_PATCHSET_REVISION \ - --notify NONE" - - exit 1 - fi + - shell: + !include-raw: ./anteater-report-to-gerrit.sh + +- builder: + name: anteater-security-audit-weekly + builders: + - shell: + !include-raw: + - ./anteater-clone-all-repos.sh + - ./anteater-security-audit-weekly.sh + diff --git a/jjb/compass4nfv/compass-build.sh b/jjb/compass4nfv/compass-build.sh index 093debba7..673a9f106 100644 --- a/jjb/compass4nfv/compass-build.sh +++ b/jjb/compass4nfv/compass-build.sh @@ -24,7 +24,16 @@ then fi cd $WORKSPACE/ -./build.sh --iso-dir $BUILD_DIRECTORY/ --iso-name compass.iso -c $CACHE_DIRECTORY + +if [[ "$BRANCH" == 'stable/danube' ]]; then + ./build.sh --iso-dir $BUILD_DIRECTORY/ --iso-name compass.iso -c $CACHE_DIRECTORY + OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/compass.iso | cut -d' ' -f1) + OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso +else + ./build.sh --tar-dir $BUILD_DIRECTORY/ --tar-name compass.tar.gz -c $CACHE_DIRECTORY + OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/compass.tar.gz | cut -d' ' -f1) + OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz +fi # list the build artifacts ls -al $BUILD_DIRECTORY @@ -34,8 +43,8 @@ ls -al $BUILD_DIRECTORY echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" - echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" - echo "OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/compass.iso | cut -d' ' -f1)" + echo "OPNFV_ARTIFACT_URL=$OPNFV_ARTIFACT_URL" + echo "OPNFV_ARTIFACT_SHA512SUM=$OPNFV_ARTIFACT_SHA512SUM" echo "OPNFV_BUILD_URL=$BUILD_URL" ) > $BUILD_DIRECTORY/opnfv.properties echo diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index e1e760dab..ddd90b6e4 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -11,11 +11,13 @@ branch: '{stream}' gs-pathname: '' disabled: false + openstack-version: ocata danube: &danube stream: danube branch: 'stable/{stream}' gs-pathname: '/{stream}' disabled: false + openstack-version: newton #-------------------------------- # POD, INSTALLER, AND BRANCH MAPPING #-------------------------------- @@ -23,19 +25,19 @@ #-------------------------------- pod: - baremetal: - slave-label: compass-baremetal + slave-label: compass-baremetal-master os-version: 'xenial' <<: *master - virtual: - slave-label: compass-virtual + slave-label: compass-virtual-master os-version: 'xenial' <<: *master - baremetal: - slave-label: compass-baremetal + slave-label: compass-baremetal-branch os-version: 'xenial' <<: *danube - virtual: - slave-label: compass-virtual + slave-label: compass-virtual-branch os-version: 'xenial' <<: *danube #-------------------------------- @@ -107,6 +109,7 @@ wrappers: - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - fix-workspace-permissions parameters: - project-parameter: @@ -133,6 +136,7 @@ predefined-parameters: | DEPLOY_SCENARIO={scenario} COMPASS_OS_VERSION={os-version} + COMPASS_OPENSTACK_VERSION={openstack-version} same-node: true block: true - trigger-builds: @@ -159,11 +163,55 @@ unstable-threshold: 'FAILURE' # dovetail only master by now, not sync with A/B/C branches # here the stream means the SUT stream, dovetail stream is defined in its own job - # only run on os-(nosdn|odl_l2)-(nofeature|bgpvpn)-ha scenario + # only run on os-(nosdn|odl_l2|onos|odl_l3)-nofeature-ha scenario + # run against SUT master branch, dovetail docker image with latest tag + # run against SUT danube branch, dovetail docker image with latest tag(odd days)and cvp.X.X.X tag(even days) - conditional-step: - condition-kind: regex-match - regex: os-(nosdn|odl_l2)-(nofeature|bgpvpn)-ha - label: '{scenario}' + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: danube + label: '{stream}' + - condition-kind: regex-match + regex: os-(nosdn|odl_l2|onos|odl_l3)-nofeature-ha + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + MON: true + WED: true + FRI: true + SUN: true + use-build-time: true + steps: + - trigger-builds: + - project: 'dovetail-compass-{pod}-proposed_tests-{stream}' + current-parameters: false + predefined-parameters: | + DOCKER_TAG=latest + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - conditional-step: + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: danube + label: '{stream}' + - condition-kind: regex-match + regex: os-(nosdn|odl_l2|onos|odl_l3)-nofeature-ha + label: '{scenario}' + - condition-kind: day-of-week + day-selector: select-days + days: + TUES: true + THURS: true + SAT: true + use-build-time: true steps: - trigger-builds: - project: 'dovetail-compass-{pod}-proposed_tests-{stream}' @@ -176,6 +224,18 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + - conditional-step: + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: os-(nosdn|odl_l2|onos|odl_l3)-nofeature-ha + label: '{scenario}' + - condition-kind: regex-match + regex: master + label: '{stream}' + steps: + - trigger-builds: + - project: 'dovetail-compass-{pod}-proposed_tests-{stream}' - job-template: name: 'compass-deploy-{pod}-daily-{stream}' @@ -201,8 +261,9 @@ - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' - timeout: - timeout: 120 + timeout: 240 abort: true + - fix-workspace-permissions parameters: - project-parameter: @@ -217,11 +278,6 @@ scm: - git-scm - wrappers: - - build-name: - name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' - - builders: - description-setter: description: "POD: $NODE_NAME" @@ -244,10 +300,6 @@ name: GS_URL default: '$GS_BASE{gs-pathname}' description: "URL to Google Storage." - - choice: - name: COMPASS_OPENSTACK_VERSION - choices: - - 'newton' ######################## # trigger macros @@ -271,15 +323,15 @@ - trigger: name: 'compass-os-onos-nofeature-ha-baremetal-centos-master-trigger' triggers: - - timed: '0 7 * * *' + - timed: '' - trigger: name: 'compass-os-ocl-nofeature-ha-baremetal-centos-master-trigger' triggers: - - timed: '0 11 * * *' + - timed: '' - trigger: name: 'compass-os-onos-sfc-ha-baremetal-centos-master-trigger' triggers: - - timed: '0 3 * * *' + - timed: '' - trigger: name: 'compass-os-odl_l2-moon-ha-baremetal-centos-master-trigger' triggers: @@ -296,7 +348,7 @@ - trigger: name: 'compass-os-nosdn-openo-ha-baremetal-master-trigger' triggers: - - timed: '0 3 * * *' + - timed: '' - trigger: name: 'compass-os-odl_l2-nofeature-ha-baremetal-master-trigger' triggers: @@ -308,15 +360,15 @@ - trigger: name: 'compass-os-onos-nofeature-ha-baremetal-master-trigger' triggers: - - timed: '0 14 * * *' + - timed: '' - trigger: name: 'compass-os-ocl-nofeature-ha-baremetal-master-trigger' triggers: - - timed: '0 10 * * *' + - timed: '' - trigger: name: 'compass-os-onos-sfc-ha-baremetal-master-trigger' triggers: - - timed: '0 6 * * *' + - timed: '' - trigger: name: 'compass-os-odl_l2-moon-ha-baremetal-master-trigger' triggers: @@ -349,11 +401,11 @@ - trigger: name: 'compass-os-ocl-nofeature-ha-baremetal-danube-trigger' triggers: - - timed: '0 5 * * *' + - timed: '' - trigger: name: 'compass-os-onos-sfc-ha-baremetal-danube-trigger' triggers: - - timed: '' + - timed: '0 5 * * *' - trigger: name: 'compass-os-odl_l2-moon-ha-baremetal-danube-trigger' triggers: @@ -370,7 +422,7 @@ - trigger: name: 'compass-os-nosdn-openo-ha-virtual-master-trigger' triggers: - - timed: '0 22 * * *' + - timed: '' - trigger: name: 'compass-os-odl_l2-nofeature-ha-virtual-master-trigger' triggers: @@ -382,19 +434,19 @@ - trigger: name: 'compass-os-onos-nofeature-ha-virtual-master-trigger' triggers: - - timed: '0 18 * * *' + - timed: '' - trigger: name: 'compass-os-ocl-nofeature-ha-virtual-master-trigger' triggers: - - timed: '0 16 * * *' + - timed: '' - trigger: name: 'compass-os-onos-sfc-ha-virtual-master-trigger' triggers: - - timed: '0 15 * * *' + - timed: '' - trigger: name: 'compass-os-odl_l2-moon-ha-virtual-master-trigger' triggers: - - timed: '0 14 * * *' + - timed: '' - trigger: name: 'compass-os-nosdn-kvm-ha-virtual-master-trigger' triggers: diff --git a/jjb/compass4nfv/compass-deploy.sh b/jjb/compass4nfv/compass-deploy.sh index 534e17e62..2668ccdf8 100644 --- a/jjb/compass4nfv/compass-deploy.sh +++ b/jjb/compass4nfv/compass-deploy.sh @@ -23,7 +23,11 @@ fi echo 1 > /proc/sys/vm/drop_caches export CONFDIR=$WORKSPACE/deploy/conf -export ISO_URL=file://$BUILD_DIRECTORY/compass.iso +if [[ "$BRANCH" = 'stable/danube' ]]; then + export ISO_URL=file://$BUILD_DIRECTORY/compass.iso +else + export ISO_URL=file://$BUILD_DIRECTORY/compass.tar.gz +fi cd $WORKSPACE diff --git a/jjb/compass4nfv/compass-dovetail-jobs.yml b/jjb/compass4nfv/compass-dovetail-jobs.yml index c321655d7..67d1e4eee 100644 --- a/jjb/compass4nfv/compass-dovetail-jobs.yml +++ b/jjb/compass4nfv/compass-dovetail-jobs.yml @@ -55,6 +55,7 @@ wrappers: - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - fix-workspace-permissions triggers: - '{auto-trigger-name}' @@ -135,8 +136,9 @@ - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' - timeout: - timeout: 120 + timeout: 240 abort: true + - fix-workspace-permissions parameters: - project-parameter: @@ -151,11 +153,6 @@ scm: - git-scm - wrappers: - - build-name: - name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' - - builders: - description-setter: description: "POD: $NODE_NAME" diff --git a/jjb/compass4nfv/compass-download-artifact.sh b/jjb/compass4nfv/compass-download-artifact.sh index 5a63c4aeb..f8915643f 100644 --- a/jjb/compass4nfv/compass-download-artifact.sh +++ b/jjb/compass4nfv/compass-download-artifact.sh @@ -18,12 +18,18 @@ curl -s -o $BUILD_DIRECTORY/latest.properties http://$GS_URL/latest.properties # source the file so we get OPNFV vars source $BUILD_DIRECTORY/latest.properties -# download the file -curl -s -o $BUILD_DIRECTORY/compass.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 - -# list the file -ls -al $BUILD_DIRECTORY/compass.iso +if [[ "$BRANCH" == 'stable/danube' ]]; then + # download the file + curl -s -o $BUILD_DIRECTORY/compass.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 + # list the file + ls -al $BUILD_DIRECTORY/compass.iso +else + # download the file + curl -s -o $BUILD_DIRECTORY/compass.tar.gz http://$OPNFV_ARTIFACT_URL > gsutil.tar.gz.log 2>&1 + # list the file + ls -al $BUILD_DIRECTORY/compass.tar.gz +fi echo echo "--------------------------------------------------------" -echo "Done!"
\ No newline at end of file +echo "Done!" diff --git a/jjb/compass4nfv/compass-upload-artifact.sh b/jjb/compass4nfv/compass-upload-artifact.sh index 73b7f07fa..87a9334b2 100644 --- a/jjb/compass4nfv/compass-upload-artifact.sh +++ b/jjb/compass4nfv/compass-upload-artifact.sh @@ -7,6 +7,11 @@ echo "Uploading the $INSTALLER_TYPE artifact. This could take some time..." echo "--------------------------------------------------------" echo +if [[ "$BRANCH" == 'stable/danube' ]]; then + FILETYPE='iso' +else + FILETYPE='tar.gz' +fi # source the opnfv.properties to get ARTIFACT_VERSION source $BUILD_DIRECTORY/opnfv.properties @@ -23,16 +28,16 @@ signiso () { time gpg2 -vvv --batch --yes --no-tty \ --default-key opnfv-helpdesk@rt.linuxfoundation.org \ --passphrase besteffort \ - --detach-sig $BUILD_DIRECTORY/compass.iso + --detach-sig $BUILD_DIRECTORY/compass.$FILETYPE -gsutil cp $BUILD_DIRECTORY/compass.iso.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso.sig +gsutil cp $BUILD_DIRECTORY/compass.$FILETYPE.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.$FILETYPE.sig echo "ISO signature Upload Complete!" } signiso # upload artifact and additional files to google storage -gsutil cp $BUILD_DIRECTORY/compass.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 +gsutil cp $BUILD_DIRECTORY/compass.$FILETYPE gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.$FILETYPE > gsutil.$FILETYPE.log 2>&1 gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 @@ -44,19 +49,19 @@ gsutil -m setmeta \ gsutil -m setmeta \ -h "Cache-Control:private, max-age=0, no-transform" \ - gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.$FILETYPE > /dev/null 2>&1 # disabled errexit due to gsutil setmeta complaints # BadRequestException: 400 Invalid argument # check if we uploaded the file successfully to see if things are fine -gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.$FILETYPE > /dev/null 2>&1 if [[ $? -ne 0 ]]; then echo "Problem while uploading artifact!" - echo "Check log $WORKSPACE/gsutil.iso.log on the machine where this build is done." + echo "Check log $WORKSPACE/gsutil.$FILETYPE.log on the machine where this build is done." exit 1 fi echo echo "--------------------------------------------------------" echo "Done!" -echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.$FILETYPE" diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index 56f54d838..f016b91c7 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -13,11 +13,15 @@ gs-pathname: '' ppa-pathname: '/{stream}' disabled: false + openstack-version: 'ocata' + branch-type: 'master' - danube: branch: 'stable/{stream}' gs-pathname: '/{stream}' ppa-pathname: '/{stream}' disabled: false + openstack-version: 'newton' + branch-type: 'branch' distro: - 'xenial': @@ -25,7 +29,7 @@ os-version: 'xenial' openstack-os-version: '' - 'centos7': - disabled: false + disabled: true os-version: 'centos7' openstack-os-version: '' ##################################### @@ -67,14 +71,15 @@ - 'compass-os-.*?-virtual-daily-.*?' block-level: 'NODE' - scm: - - git-scm-gerrit - wrappers: - ssh-agent-wrapper - timeout: - timeout: 120 + timeout: 240 fail: true + - fix-workspace-permissions + + scm: + - git-scm-gerrit triggers: - gerrit: @@ -108,7 +113,7 @@ - project-parameter: project: '{project}' branch: '{branch}' - - 'compass-virtual-defaults' + - 'compass-virtual-{branch-type}-defaults' - '{installer}-defaults' - 'compass-verify-defaults': installer: '{installer}' @@ -143,6 +148,7 @@ current-parameters: true predefined-parameters: | COMPASS_OS_VERSION={os-version} + COMPASS_OPENSTACK_VERSION={openstack-version} node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -151,14 +157,14 @@ condition: SUCCESSFUL projects: - name: 'functest-compass-virtual-suite-{stream}' - current-parameters: true + current-parameters: false predefined-parameters: FUNCTEST_SUITE_NAME=healthcheck node-parameters: true kill-phase-on: NEVER abort-all-job: true - name: 'functest-compass-virtual-suite-{stream}' - current-parameters: true + current-parameters: false predefined-parameters: FUNCTEST_SUITE_NAME=vping_ssh node-parameters: true @@ -188,14 +194,15 @@ - 'compass-os-.*?-virtual-daily-.*?' block-level: 'NODE' - scm: - - git-scm-gerrit - wrappers: - ssh-agent-wrapper - timeout: - timeout: 120 + timeout: 240 fail: true + - fix-workspace-permissions + + scm: + - git-scm-gerrit triggers: - gerrit: @@ -227,7 +234,7 @@ - project-parameter: project: '{project}' branch: '{branch}' - - 'compass-virtual-defaults' + - 'compass-virtual-{branch-type}-defaults' - '{installer}-defaults' - 'compass-verify-defaults': installer: '{installer}' @@ -287,14 +294,15 @@ - 'functest-compass-virtual.*' block-level: 'NODE' - scm: - - git-scm-gerrit - wrappers: - ssh-agent-wrapper - timeout: - timeout: 120 + timeout: 240 fail: true + - fix-workspace-permissions + + scm: + - git-scm-gerrit builders: - description-setter: @@ -344,10 +352,6 @@ name: PPA_CACHE default: "$WORKSPACE/work/repo/" - choice: - name: COMPASS_OPENSTACK_VERSION - choices: - - 'newton' - - choice: name: COMPASS_OS_VERSION choices: - 'xenial' diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index aac76baa4..592e54dc5 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -45,6 +45,12 @@ # NOHA scenarios - 'os-nosdn-nofeature-noha': auto-trigger-name: 'daisy-{scenario}-{pod}-daily-{stream}-trigger' + # ODL_L3 scenarios + - 'os-odl_l3-nofeature-noha': + auto-trigger-name: 'daisy-{scenario}-{pod}-daily-{stream}-trigger' + # ODL_L2 scenarios + - 'os-odl_l2-nofeature-noha': + auto-trigger-name: 'daisy-{scenario}-{pod}-daily-{stream}-trigger' jobs: - '{project}-{scenario}-{pod}-daily-{stream}' @@ -144,7 +150,7 @@ installer: '{installer}' - string: name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' + default: 'os-nosdn-nofeature-noha' - 'daisy-project-parameter': gs-pathname: '{gs-pathname}' - string: @@ -183,6 +189,16 @@ - trigger: name: 'daisy-os-nosdn-nofeature-noha-baremetal-daily-master-trigger' triggers: + - timed: 'H 12 * * *' +# ODL_L3 Scenarios +- trigger: + name: 'daisy-os-odl_l3-nofeature-noha-baremetal-daily-master-trigger' + triggers: + - timed: 'H 16 * * *' +# ODL_L2 Scenarios +- trigger: + name: 'daisy-os-odl_l2-nofeature-noha-baremetal-daily-master-trigger' + triggers: - timed: '' #----------------------------------------------- # Triggers for job running on daisy-virtual against master branch @@ -195,5 +211,15 @@ - trigger: name: 'daisy-os-nosdn-nofeature-noha-virtual-daily-master-trigger' triggers: - - timed: 'H 8,22 * * *' + - timed: 'H 12 * * *' +# ODL_L3 Scenarios +- trigger: + name: 'daisy-os-odl_l3-nofeature-noha-virtual-daily-master-trigger' + triggers: + - timed: 'H 16 * * *' +# ODL_L3 Scenarios +- trigger: + name: 'daisy-os-odl_l2-nofeature-noha-virtual-daily-master-trigger' + triggers: + - timed: '' diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index 0a9d43d25..57e44e314 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -20,10 +20,6 @@ branch: '{stream}' gs-pathname: '' disabled: false - - danube: - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false phase: - 'build': @@ -64,7 +60,7 @@ - git-scm triggers: - - timed: '0 H/8 * * *' + - timed: '0 8 * * *' parameters: - project-parameter: @@ -165,7 +161,7 @@ description: 'Git URL to use on this Jenkins Slave' - string: name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' + default: 'os-nosdn-nofeature-noha' - '{installer}-project-parameter': gs-pathname: '{gs-pathname}' diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 807d436da..c5454c799 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -17,11 +17,9 @@ installer: - apex: - slave-label: 'ool-virtual1' - pod: 'ool-virtual1' + slave-label: 'doctor-apex-verify' - fuel: - slave-label: 'ool-virtual2' - pod: 'ool-virtual2' + slave-label: 'doctor-fuel-verify' #- joid: # slave-label: 'ool-virtual3' # pod: 'ool-virtual3' @@ -34,12 +32,25 @@ - verify: profiler: 'none' auto-trigger-name: 'doctor-verify' + is-python: false - profiling: profiler: 'poc' auto-trigger-name: 'experimental' + is-python: false + - python-verify: + profiler: 'none' + auto-trigger-name: 'doctor-verify' + is-python: true + + pod: + - arm-pod2: + slave-label: '{pod}' + - arm-pod3: + slave-label: '{pod}' jobs: - 'doctor-verify-{stream}' + - 'doctor-{task}-{installer}-{inspector}-{pod}-{stream}' - 'doctor-{task}-{installer}-{inspector}-{stream}' - job-template: @@ -83,6 +94,32 @@ - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" - job-template: + name: 'doctor-{task}-{installer}-{inspector}-{pod}-{stream}' + + node: '{slave-label}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + + triggers: + - '{auto-trigger-name}': + project: '{project}' + branch: '{branch}' + + builders: + - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" + + +- job-template: name: 'doctor-{task}-{installer}-{inspector}-{stream}' node: '{slave-label}' @@ -115,7 +152,7 @@ default: 'doctor-notification' - string: name: TESTCASE_OPTIONS - default: '-e INSPECTOR_TYPE={inspector} -e PROFILER_TYPE={profiler} -v $WORKSPACE:/home/opnfv/repos/doctor' + default: '-e INSPECTOR_TYPE={inspector} -e PROFILER_TYPE={profiler} -e PYTHON_ENABLE={is-python} -v $WORKSPACE:/home/opnfv/repos/doctor' description: 'Addtional parameters specific to test case(s)' # functest-parameter - string: diff --git a/jjb/dovetail/dovetail-ci-jobs.yml b/jjb/dovetail/dovetail-ci-jobs.yml index 682948d8b..43978f6dc 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yml +++ b/jjb/dovetail/dovetail-ci-jobs.yml @@ -25,7 +25,7 @@ branch: 'stable/{stream}' dovetail-branch: master gs-pathname: '/{stream}' - docker-tag: 'latest' + docker-tag: 'cvp.0.2.0' #----------------------------------- # POD, PLATFORM, AND BRANCH MAPPING @@ -229,6 +229,10 @@ name: DOVETAIL_REPO_DIR default: "/home/opnfv/dovetail" description: "Directory where the dovetail repository is cloned" + - string: + name: SUT_BRANCH + default: '{branch}' + description: "SUT branch" scm: - git-scm diff --git a/jjb/dovetail/dovetail-cleanup.sh b/jjb/dovetail/dovetail-cleanup.sh index 0ee789a97..3ae0cbcc9 100755 --- a/jjb/dovetail/dovetail-cleanup.sh +++ b/jjb/dovetail/dovetail-cleanup.sh @@ -1,4 +1,11 @@ #!/bin/bash +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index dce7e5862..bf96fd492 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -1,4 +1,11 @@ #!/bin/bash +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## #the noun INSTALLER is used in community, here is just the example to run. #multi-platforms are supported. @@ -7,14 +14,12 @@ set -e [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" DOVETAIL_HOME=${WORKSPACE}/cvp -if [ -d ${DOVETAIL_HOME} ]; then - sudo rm -rf ${DOVETAIL_HOME}/* -else - sudo mkdir -p ${DOVETAIL_HOME} -fi +[ -d ${DOVETAIL_HOME} ] && sudo rm -rf ${DOVETAIL_HOME} + +mkdir -p ${DOVETAIL_HOME} DOVETAIL_CONFIG=${DOVETAIL_HOME}/pre_config -sudo mkdir -p ${DOVETAIL_CONFIG} +mkdir -p ${DOVETAIL_CONFIG} sshkey="" # The path of openrc.sh is defined in fetch_os_creds.sh @@ -47,7 +52,12 @@ releng_repo=${WORKSPACE}/releng git clone https://gerrit.opnfv.org/gerrit/releng ${releng_repo} >/dev/null if [[ ${INSTALLER_TYPE} != 'joid' ]]; then - sudo /bin/bash ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} + echo "SUT branch is $SUT_BRANCH" + echo "dovetail branch is $BRANCH" + BRANCH_BACKUP=$BRANCH + export BRANCH=$SUT_BRANCH + ${releng_repo}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} + export BRANCH=$BRANCH_BACKUP fi if [[ -f $OPENRC ]]; then @@ -59,6 +69,8 @@ else exit 1 fi +set +e + sudo pip install virtualenv cd ${releng_repo}/modules @@ -83,6 +95,8 @@ ${cmd} deactivate +set -e + cd ${WORKSPACE} if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then @@ -102,7 +116,8 @@ if [ "$INSTALLER_TYPE" == "fuel" ]; then fi # sdnvpn test case needs to download this image first before running -sudo wget -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${DOVETAIL_CONFIG} +echo "Download image ubuntu-16.04-server-cloudimg-amd64-disk1.img ..." +wget -q -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${DOVETAIL_CONFIG} opts="--privileged=true -id" diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index dd0590c72..68677089d 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -293,6 +293,10 @@ name: GS_URL default: artifacts.opnfv.org/$PROJECT{gs-pathname} description: "URL to Google Storage." + - string: + name: SSH_KEY + default: "/tmp/mcp.rsa" + description: "Path to private SSH key to access environment nodes. For MCP deployments only." ######################## # trigger macros ######################## diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index 4d48ee587..2fb5c71e4 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -12,11 +12,13 @@ set -o pipefail export TERM="vt220" -# source the file so we get OPNFV vars -source latest.properties +if [[ "$BRANCH" != 'master' ]]; then + # source the file so we get OPNFV vars + source latest.properties -# echo the info about artifact that is used during the deployment -echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment" + # echo the info about artifact that is used during the deployment + echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment" +fi if [[ "$JOB_NAME" =~ "merge" ]]; then # set simplest scenario for virtual deploys to run for merges @@ -75,7 +77,7 @@ echo "--------------------------------------------------------" echo "Scenario: $DEPLOY_SCENARIO" echo "Lab: $LAB_NAME" echo "POD: $POD_NAME" -echo "ISO: ${OPNFV_ARTIFACT_URL/*\/}" +[[ "$BRANCH" != 'master' ]] && echo "ISO: ${OPNFV_ARTIFACT_URL/*\/}" echo echo "Starting the deployment using $INSTALLER_TYPE. This could take some time..." echo "--------------------------------------------------------" diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh index 8cc552e8d..c3b8253de 100755 --- a/jjb/fuel/fuel-download-artifact.sh +++ b/jjb/fuel/fuel-download-artifact.sh @@ -10,6 +10,9 @@ set -o errexit set -o pipefail +# disable Fuel ISO download for master branch +[[ "$BRANCH" == 'master' ]] && exit 0 + # use proxy url to replace the nomral URL, for googleusercontent.com will be blocked randomly [[ "$NODE_NAME" =~ (zte) ]] && GS_URL=${GS_BASE_PROXY%%/*}/$GS_URL diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index 8de092d29..fdef6f483 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -203,10 +203,10 @@ installer: fuel <<: *danube # PODs for verify jobs triggered by each patch upload - - ool-virtual1: - slave-label: '{pod}' - installer: apex - <<: *master +# - ool-virtual1: +# slave-label: '{pod}' +# installer: apex +# <<: *master #-------------------------------- testsuite: diff --git a/jjb/functest/functest-loop.sh b/jjb/functest/functest-loop.sh index 676890644..00a5f135b 100755 --- a/jjb/functest/functest-loop.sh +++ b/jjb/functest/functest-loop.sh @@ -2,7 +2,11 @@ set +e [[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+="-r" -cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/run_tests.py -t all ${flags}" +if [ "$BRANCH" == 'master' ]; then + cmd="run_tests -t all ${flags}" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/run_tests.py -t all ${flags}" +fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd diff --git a/jjb/functest/functest-project-jobs.yml b/jjb/functest/functest-project-jobs.yml index 7036f20c0..353423d59 100644 --- a/jjb/functest/functest-project-jobs.yml +++ b/jjb/functest/functest-project-jobs.yml @@ -9,6 +9,7 @@ jobs: - 'functest-verify-{stream}' + - 'functest-docs-upload-{stream}' stream: - master: @@ -18,7 +19,7 @@ - danube: branch: 'stable/{stream}' gs-pathname: '/{stream}' - disabled: false + disabled: true - job-template: name: 'functest-verify-{stream}' @@ -80,6 +81,41 @@ unhealthy: 40 failing: 30 +- job-template: + name: 'functest-docs-upload-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - functest-upload-doc-artifact + ################################ # job builders ################################ @@ -89,3 +125,10 @@ builders: - shell: | cd $WORKSPACE && tox + +- builder: + name: functest-upload-doc-artifact + builders: + - shell: | + cd $WORKSPACE && tox -edocs + wget -O - https://git.opnfv.org/releng/plain/utils/upload-artifact.sh | bash -s "functest/api/_build" "docs" diff --git a/jjb/functest/functest-suite.sh b/jjb/functest/functest-suite.sh index 5d1ed28f5..9b7f1356c 100755 --- a/jjb/functest/functest-suite.sh +++ b/jjb/functest/functest-suite.sh @@ -10,7 +10,11 @@ global_ret_val=0 tests=($(echo $FUNCTEST_SUITE_NAME | tr "," "\n")) for test in ${tests[@]}; do - cmd="python /home/opnfv/repos/functest/functest/ci/run_tests.py -t $test" + if [ "$BRANCH" == 'master' ]; then + cmd="run_tests -t $test" + else + cmd="python /home/opnfv/repos/functest/functest/ci/run_tests.py -t $test" + fi docker exec $container_id $cmd let global_ret_val+=$? done diff --git a/jjb/functest/set-functest-env.sh b/jjb/functest/set-functest-env.sh index 558e2487d..5f936f5d9 100755 --- a/jjb/functest/set-functest-env.sh +++ b/jjb/functest/set-functest-env.sh @@ -90,7 +90,12 @@ if [ $(docker ps | grep "${FUNCTEST_IMAGE}:${DOCKER_TAG}" | wc -l) == 0 ]; then exit 1 fi -cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/prepare_env.py start" +if [ "$BRANCH" == 'master' ]; then + cmd="prepare_env start" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/prepare_env.py start" +fi + echo "Executing command inside the docker: ${cmd}" docker exec ${container_id} ${cmd} diff --git a/jjb/global/installer-params.yml b/jjb/global/installer-params.yml index 40fc42c76..ee154af03 100644 --- a/jjb/global/installer-params.yml +++ b/jjb/global/installer-params.yml @@ -38,6 +38,14 @@ default: '10.20.0.2' description: 'IP of the installer' - string: + name: SALT_MASTER_IP + default: '192.168.10.100' + description: 'IP of the salt master (for mcp deployments)' + - string: + name: SSH_KEY + default: '/tmp/mcp.rsa' + description: 'Path to private SSH key to access environment nodes' + - string: name: INSTALLER_TYPE default: fuel description: 'Installer used for deploying OPNFV on this POD' diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index ce8990564..92199d6e7 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -170,7 +170,7 @@ - pre-scm-buildstep: - shell: | #!/bin/bash - sudo chown -R $USER $WORKSPACE || exit 1 + sudo chown -R $USER:$USER $WORKSPACE || exit 1 - builder: name: build-html-and-pdf-docs-output @@ -259,8 +259,11 @@ find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \ sed -e "s|^$local_path| http://$gs_path|" >> gerrit_comment.txt +# To take advantage of this macro, have your build write +# out the file 'gerrit_comment.txt' with information to post +# back to gerrit and include this macro in the list of builders. - builder: - name: report-docs-build-result-to-gerrit + name: report-build-result-to-gerrit builders: - shell: | #!/bin/bash @@ -407,14 +410,14 @@ builders: - build-html-and-pdf-docs-output - upload-under-review-docs-to-opnfv-artifacts - - report-docs-build-result-to-gerrit + - report-build-result-to-gerrit - builder: name: upload-merged-docs builders: - build-html-and-pdf-docs-output - upload-generated-docs-to-opnfv-artifacts - - report-docs-build-result-to-gerrit + - report-build-result-to-gerrit - remove-old-docs-from-opnfv-artifacts - builder: diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index cdc08eaf4..50859c46f 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -154,6 +154,26 @@ default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - parameter: + name: 'compass-baremetal-master-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'compass-baremetal-master' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'compass-baremetal-branch-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'compass-baremetal-branch' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' +- parameter: name: 'fuel-baremetal-defaults' parameters: - label: @@ -206,7 +226,7 @@ default: 'daisy-baremetal' - string: name: INSTALLER_IP - default: '10.20.11.2' + default: '10.20.7.3' description: 'IP of the installer' - string: name: GIT_BASE @@ -240,6 +260,26 @@ default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - parameter: + name: 'compass-virtual-master-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'compass-virtual-master' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'compass-virtual-branch-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'compass-virtual-branch' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' +- parameter: name: 'fuel-virtual-defaults' parameters: - label: @@ -822,15 +862,13 @@ default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'ool-defaults' + name: 'doctor-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - ool-virtual1 - - ool-virtual2 - - ool-virtual3 + - '{default-slave}' default-slaves: - '{default-slave}' - string: @@ -842,20 +880,20 @@ default: /root/.ssh/id_rsa description: 'SSH key to be used' - parameter: - name: 'ool-virtual1-defaults' + name: 'doctor-apex-verify-defaults' parameters: - - 'ool-defaults': - default-slave: 'ool-virtual1' + - 'doctor-defaults': + default-slave: 'doctor-apex-verify' - parameter: - name: 'ool-virtual2-defaults' + name: 'doctor-fuel-verify-defaults' parameters: - - 'ool-defaults': - default-slave: 'ool-virtual2' + - 'doctor-defaults': + default-slave: 'doctor-fuel-verify' - parameter: - name: 'ool-virtual3-defaults' + name: 'doctor-joid-verify-defaults' parameters: - - 'ool-defaults': - default-slave: 'ool-virtual3' + - 'doctor-defaults': + default-slave: 'doctor-joid-verify' - parameter: name: 'multisite-virtual-defaults' parameters: diff --git a/jjb/netready/netready.yml b/jjb/netready/netready.yml index 9a4d8858c..2702c45b3 100644 --- a/jjb/netready/netready.yml +++ b/jjb/netready/netready.yml @@ -58,7 +58,7 @@ - job-template: name: 'netready-build-gluon-packages-daily-{stream}' - disabled: false + disabled: true concurrent: true diff --git a/jjb/releng/testapi-automate.yml b/jjb/releng/automate.yml index dd76538a3..73bef3e8b 100644 --- a/jjb/releng/testapi-automate.yml +++ b/jjb/releng/automate.yml @@ -1,20 +1,22 @@ - project: - name: testapi-automate + name: utils-automate stream: - master: branch: '{stream}' - gs-pathname: '' + + module: + - 'testapi' + - 'reporting' phase: - - 'docker-update' - 'docker-deploy': slave-label: 'testresults' - 'generate-doc' jobs: - - 'testapi-automate-{stream}' - - 'testapi-automate-{phase}-{stream}' - - 'testapi-verify-{stream}' + - '{module}-automate-{stream}' + - '{module}-automate-{phase}-{stream}' + - '{module}-verify-{stream}' project: 'releng' @@ -44,7 +46,7 @@ - mongodb-backup - job-template: - name: 'testapi-verify-{stream}' + name: '{module}-verify-{stream}' parameters: - project-parameter: @@ -76,10 +78,13 @@ branch-pattern: '**/{branch}' file-paths: - compare-type: 'ANT' - pattern: 'utils/test/testapi/**' + pattern: 'utils/test/{module}/**' builders: - - run-unit-tests + - shell: | + cd ./utils/test/{module}/ + bash run_test.sh + cp *.xml $WORKSPACE publishers: - junit: @@ -101,7 +106,7 @@ failing: 30 - job-template: - name: 'testapi-automate-{stream}' + name: '{module}-automate-{stream}' project-type: multijob @@ -118,8 +123,12 @@ branch: '{branch}' - string: name: DOCKER_TAG - default: "latest" - description: "Tag name for testapi docker image" + default: 'latest' + description: 'Tag name for {module} docker image' + - string: + name: MODULE_NAME + default: '{module}' + description: "Name of the module" - 'opnfv-build-defaults' scm: @@ -146,24 +155,17 @@ branch-pattern: '**/{branch}' file-paths: - compare-type: 'ANT' - pattern: 'utils/test/testapi/**' + pattern: 'utils/test/{module}/**' builders: - description-setter: description: "Built on $NODE_NAME" - - multijob: - name: docker-update - condition: SUCCESSFUL - projects: - - name: 'testapi-automate-docker-update-{stream}' - current-parameters: true - kill-phase-on: FAILURE - abort-all-job: true + - docker-update - multijob: name: docker-deploy condition: SUCCESSFUL projects: - - name: 'testapi-automate-docker-deploy-{stream}' + - name: '{module}-automate-docker-deploy-{stream}' current-parameters: false predefined-parameters: | GIT_BASE=$GIT_BASE @@ -175,7 +177,7 @@ name: generate-doc condition: SUCCESSFUL projects: - - name: 'testapi-automate-generate-doc-{stream}' + - name: '{module}-automate-generate-doc-{stream}' current-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -184,7 +186,7 @@ - 'email-publisher' - job-template: - name: 'testapi-automate-{phase}-{stream}' + name: '{module}-automate-{phase}-{stream}' properties: - throttle: @@ -196,10 +198,6 @@ - project-parameter: project: '{project}' branch: '{branch}' - - string: - name: DOCKER_TAG - default: "latest" - description: "Tag name for testapi docker image" wrappers: - ssh-agent-wrapper @@ -213,28 +211,16 @@ builders: - description-setter: description: "Built on $NODE_NAME" - - 'testapi-automate-{phase}-macro' + - '{module}-automate-{phase}-macro' ################################ # job builders ################################ - builder: - name: mongodb-backup - builders: - - shell: | - bash ./jjb/releng/testapi-backup-mongodb.sh - -- builder: - name: 'run-unit-tests' + name: 'docker-update' builders: - - shell: | - bash ./utils/test/testapi/run_test.sh - -- builder: - name: 'testapi-automate-docker-update-macro' - builders: - - shell: | - bash ./jjb/releng/testapi-docker-update.sh + - shell: + !include-raw: ./docker-update.sh - builder: name: 'testapi-automate-generate-doc-macro' @@ -255,10 +241,26 @@ bash ./utils/test/testapi/htmlize/push-doc-artifact.sh - builder: + name: 'reporting-automate-generate-doc-macro' + builders: + - shell: echo "To Be Done" + +- builder: name: 'testapi-automate-docker-deploy-macro' builders: - shell: | - bash ./jjb/releng/testapi-docker-deploy.sh + bash ./jjb/releng/docker-deploy.sh 'sudo docker run -dti -p 8082:8000 -e mongodb_url=mongodb://172.17.0.1:27017 -e base_url=http://testresults.opnfv.org/test opnfv/testapi' "http://testresults.opnfv.org/test/swagger/APIs" +- builder: + name: 'reporting-automate-docker-deploy-macro' + builders: + - shell: | + bash ./jjb/releng/docker-deploy.sh 'sudo docker run -itd -p 8084:8000 -e SERVER_URL=http://testresults.opnfv.org/reporting2:8084 opnfv/reporting' "http://testresults.opnfv.org/reporting2/reporting/index.html" + +- builder: + name: mongodb-backup + builders: + - shell: | + bash ./jjb/releng/testapi-backup-mongodb.sh ################################ # job publishers @@ -268,6 +270,6 @@ name: 'email-publisher' publishers: - email: - recipients: rohitsakala@gmail.com feng.xiaowei@zte.com.cn + recipients: rohitsakala@gmail.com feng.xiaowei@zte.com.cn morgan.richomme@orange.com notify-every-unstable-build: false send-to-individuals: true diff --git a/jjb/releng/testapi-docker-deploy.sh b/jjb/releng/docker-deploy.sh index b4e60b09a..b3b930fc4 100644 --- a/jjb/releng/testapi-docker-deploy.sh +++ b/jjb/releng/docker-deploy.sh @@ -1,10 +1,30 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one * +# or more contributor license agreements. See the NOTICE file * +# distributed with this work for additional information * +# regarding copyright ownership. The ASF licenses this file * +# to you under the Apache License, Version 2.0 (the * +# "License"); you may not use this file except in compliance * +# with the License. You may obtain a copy of the License at * +# * +# http://www.apache.org/licenses/LICENSE-2.0 * +# * +# Unless required by applicable law or agreed to in writing, * +# software distributed under the License is distributed on an * +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * +# KIND, either express or implied. See the License for the * +# specific language governing permissions and limitations * +# under the License. * + +# Assigning Variables +command=$1 +url=$2 function check() { # Verify hosted sleep 5 - cmd=`curl -s --head --request GET http://testresults.opnfv.org/test/swagger/spec | grep '200 OK' > /dev/null` + cmd=`curl -s --head --request GET ${url} | grep '200 OK' > /dev/null` rc=$? echo $rc @@ -63,7 +83,7 @@ else fi echo "Running a container with the new image" -sudo docker run -dti -p "8082:8000" -e "mongodb_url=mongodb://172.17.0.1:27017" -e "swagger_url=http://testresults.opnfv.org/test" opnfv/testapi:latest +$command:latest if check; then echo "TestResults Hosted." @@ -71,7 +91,7 @@ else echo "TestResults Hosting Failed" if [[ $(sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}') ]]; then echo "Running old Image" - sudo docker run -dti -p "8082:8000" -e "mongodb_url=mongodb://172.17.0.1:27017" -e "swagger_url=http://testresults.opnfv.org/test" opnfv/testapi:old + $command:old exit 1 fi fi diff --git a/jjb/releng/docker-update.sh b/jjb/releng/docker-update.sh new file mode 100644 index 000000000..559ac835c --- /dev/null +++ b/jjb/releng/docker-update.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one * +# or more contributor license agreements. See the NOTICE file * +# distributed with this work for additional information * +# regarding copyright ownership. The ASF licenses this file * +# to you under the Apache License, Version 2.0 (the * +# "License"); you may not use this file except in compliance * +# with the License. You may obtain a copy of the License at * +# * +# http://www.apache.org/licenses/LICENSE-2.0 * +# * +# Unless required by applicable law or agreed to in writing, * +# software distributed under the License is distributed on an * +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * +# KIND, either express or implied. See the License for the * +# specific language governing permissions and limitations * +# under the License. * + +set -o errexit +set -o nounset + +cd $WORKSPACE/utils/test/$MODULE_NAME/docker/ + +# Remove previous containers +docker ps -a | grep "opnfv/$MODULE_NAME" | awk '{ print $1 }' | xargs -r docker rm -f + +# Remove previous images +docker images | grep "opnfv/$MODULE_NAME" | awk '{ print $3 }' | xargs -r docker rmi -f + +# Start build +docker build --no-cache -t opnfv/$MODULE_NAME:$DOCKER_TAG . + +# Push Image +docker push opnfv/$MODULE_NAME:$DOCKER_TAG diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index 2aa52adc5..ebd0c9f3d 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -73,6 +73,8 @@ fi # Get tag version echo "Current branch: $BRANCH" +BUILD_BRANCH=$BRANCH + if [[ "$BRANCH" == "master" ]]; then DOCKER_TAG="latest" elif [[ -n "${RELEASE_VERSION-}" ]]; then @@ -82,19 +84,17 @@ else DOCKER_TAG="stable" fi +if [[ -n "${COMMIT_ID-}" && -n "${RELEASE_VERSION-}" ]]; then + DOCKER_TAG=$RELEASE_VERSION + BUILD_BRANCH=$COMMIT_ID +fi + # Start the build echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" echo "--------------------------------------------------------" echo -if [[ $DOCKER_REPO_NAME == *"dovetail"* ]]; then - if [[ -n "${RELEASE_VERSION-}" ]]; then - DOCKER_TAG=${RELEASE_VERSION} - fi - cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG -f $DOCKERFILE ." -else - cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BRANCH - -f $DOCKERFILE ." -fi +cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH + -f $DOCKERFILE ." echo ${cmd} ${cmd} diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 5fe0eb913..095ba4129 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -106,6 +106,10 @@ default: "opnfv/{project}" description: "Dockerhub repo to be pushed to." - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: name: RELEASE_VERSION default: "" description: "Release version, e.g. 1.0, 2.0, 3.0" diff --git a/jjb/releng/testapi-docker-update.sh b/jjb/releng/testapi-docker-update.sh deleted file mode 100644 index 84f5c3217..000000000 --- a/jjb/releng/testapi-docker-update.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset - -cd $WORKSPACE/utils/test/testapi/docker/ - -# Remove previous containers -docker ps -a | grep "opnfv/testapi" | awk '{ print $1 }' | xargs -r docker rm -f - -# Remove previous images -docker images | grep "opnfv/testapi" | awk '{ print $3 }' | xargs -r docker rmi -f - -# Start build -docker build --no-cache -t opnfv/testapi:$DOCKER_TAG . - -# Push Image -docker push opnfv/testapi:$DOCKER_TAG diff --git a/jjb/storperf/storperf.yml b/jjb/storperf/storperf.yml index cd40050b2..13186a1ad 100644 --- a/jjb/storperf/storperf.yml +++ b/jjb/storperf/storperf.yml @@ -158,12 +158,50 @@ name: DOCKER_TAG default: '{docker-tag}' description: 'Tag to pull docker image' + - choice: + name: DISK_TYPE + choices: + - 'SSD' + - 'HDD' + default: 'HDD' + description: 'The type of hard disk that Cinder uses' + - string: + name: AGENT_COUNT + description: 'The number of slave agents to start. Defaults to the cinder node count' + - string: + name: VOLUME_SIZE + default: '4' + description: 'Size of Cinder volume (in GB)' + - string: + name: WORKLOADS + default: 'wr,rr,rw' + description: 'Workloads to run' + - string: + name: BLOCK_SIZES + default: '2048,16384' + description: 'Block sizes for VM I/O operations' + - string: + name: QUEUE_DEPTHS + default: '1,4' + description: 'Number of simultaneous I/O operations to keep active' + - string: + name: STEADY_STATE_SAMPLES + default: '10' + description: 'Number of samples to use (1 per minute) to measure steady state' + - string: + name: DEADLINE + description: 'Maximum run time in minutes if steady state cannot be found. Defaults to 3 times steady state samples' + - choice: + name: TEST_CASE + choices: + - 'snia_steady_state' + description: 'The test case to run' scm: - git-scm triggers: - - timed: 'H H * * *' + - timed: '0 22 * * *' builders: - shell: | diff --git a/jjb/xci/bifrost-periodic-jobs.yml b/jjb/xci/bifrost-periodic-jobs.yml index 3e9ff678e..9773cfd9e 100644 --- a/jjb/xci/bifrost-periodic-jobs.yml +++ b/jjb/xci/bifrost-periodic-jobs.yml @@ -135,7 +135,7 @@ # trigger is disabled until we know which jobs we will have # and adjust stuff accordingly triggers: - - timed: '#@midnight' + - timed: '' # '@midnight' builders: - description-setter: diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml index c6da9f413..319f8eb28 100644 --- a/jjb/xci/bifrost-verify-jobs.yml +++ b/jjb/xci/bifrost-verify-jobs.yml @@ -20,9 +20,9 @@ # distros #-------------------------------- distro: - - 'trusty': + - 'xenial': disabled: false - dib-os-release: 'trusty' + dib-os-release: 'xenial' dib-os-element: 'ubuntu-minimal' dib-os-packages: 'vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' extra-dib-elements: 'openssh-server' diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh index 2e6f2272c..b522b8969 100755 --- a/jjb/xci/bifrost-verify.sh +++ b/jjb/xci/bifrost-verify.sh @@ -89,7 +89,7 @@ function cleanup_and_upload() { } # check distro to see if we support it -if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then +if [[ ! "$DISTRO" =~ (xenial|centos7|suse) ]]; then echo "Distro $DISTRO is not supported!" exit 1 fi diff --git a/jjb/xci/osa-periodic-jobs.yml b/jjb/xci/osa-periodic-jobs.yml index 56a4b18b4..722b077ac 100644 --- a/jjb/xci/osa-periodic-jobs.yml +++ b/jjb/xci/osa-periodic-jobs.yml @@ -132,7 +132,7 @@ # trigger is disabled until we know which jobs we will have # and adjust stuff accordingly triggers: - - timed: '#@midnight' + - timed: '' # '@midnight' builders: - description-setter: diff --git a/jjb/yardstick/yardstick-ci-jobs.yml b/jjb/yardstick/yardstick-daily-jobs.yml index 5ff36f842..ff1d47eb4 100644 --- a/jjb/yardstick/yardstick-ci-jobs.yml +++ b/jjb/yardstick/yardstick-daily-jobs.yml @@ -282,7 +282,7 @@ publishers: - email: - recipients: jean.gaoliang@huawei.com limingjiang@huawei.com + recipients: jean.gaoliang@huawei.com limingjiang@huawei.com ross.b.brattain@intel.com ######################## # builder macros diff --git a/jjb/yardstick/yardstick-daily.sh b/jjb/yardstick/yardstick-daily.sh index 973f83ad5..1c2abad3f 100755 --- a/jjb/yardstick/yardstick-daily.sh +++ b/jjb/yardstick/yardstick-daily.sh @@ -31,7 +31,8 @@ fi opts="--privileged=true --rm" envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ -e NODE_NAME=${NODE_NAME} -e EXTERNAL_NETWORK=${EXTERNAL_NETWORK} \ - -e YARDSTICK_BRANCH=${BRANCH} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" + -e YARDSTICK_BRANCH=${BRANCH} -e BRANCH=${BRANCH} \ + -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" # Pull the image with correct tag echo "Yardstick: Pulling image opnfv/yardstick:${DOCKER_TAG}" |