diff options
25 files changed, 320 insertions, 122 deletions
diff --git a/docs/infra/jenkins/jjb-usage.rst b/docs/infra/jenkins/jjb-usage.rst index 671278121..d467dbe8a 100644 --- a/docs/infra/jenkins/jjb-usage.rst +++ b/docs/infra/jenkins/jjb-usage.rst @@ -10,7 +10,7 @@ Creating/Configuring/Verifying Jenkins Jobs Clone and setup the repo:: - git clone ssh://YOU@gerrit.opnfv.org:29418/releng + git clone --recursive ssh://YOU@gerrit.opnfv.org:29418/releng cd releng git review -s @@ -29,13 +29,13 @@ Make changes:: Test with tox:: - tox -v -ejjb + tox -e jjb .. note:: You can also test the jobs under a single jjb directory by specifying the directory. For example to test only the releng jobs, you could run: - tox -v -e jjb -- jjb/releng + tox -e jjb -- jjb/global:jjb/global-jjb:jjb/releng Submit the change to gerrit:: diff --git a/jjb/3rd_party_ci/download-netvirt-artifact.sh b/jjb/3rd_party_ci/download-netvirt-artifact.sh index 7ecf8d78d..5a50e8a93 100755 --- a/jjb/3rd_party_ci/download-netvirt-artifact.sh +++ b/jjb/3rd_party_ci/download-netvirt-artifact.sh @@ -10,7 +10,7 @@ if [ "$ODL_BRANCH" != 'master' ]; then DIST=$(echo ${ODL_BRANCH} | sed -rn 's#([a-zA-Z]+)/([a-zA-Z]+)#\2#p') ODL_BRANCH=$(echo ${ODL_BRANCH} | sed -rn 's#([a-zA-Z]+)/([a-zA-Z]+)#\1%2F\2#p') else - DIST='nitrogen' + DIST='flourine' fi CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~${ODL_BRANCH}~${GERRIT_CHANGE_ID}/detail" # due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details diff --git a/jjb/3rd_party_ci/odl-netvirt.yaml b/jjb/3rd_party_ci/odl-netvirt.yaml index 863eb94dd..c077fce2c 100644 --- a/jjb/3rd_party_ci/odl-netvirt.yaml +++ b/jjb/3rd_party_ci/odl-netvirt.yaml @@ -13,8 +13,8 @@ branch: '{stream}' gs-pathname: '' disabled: false - - carbon: - branch: 'stable/carbon' + - oxygen: + branch: 'stable/oxygen' gs-pathname: '' disabled: false ##################################### @@ -22,11 +22,11 @@ ##################################### phase: - 'create-apex-vms': - slave-label: 'odl-netvirt-virtual-intel' + slave-label: 'apex-virtual-master' - 'install-netvirt': - slave-label: 'odl-netvirt-virtual-intel' + slave-label: 'apex-virtual-master' - 'postprocess': - slave-label: 'odl-netvirt-virtual-intel' + slave-label: 'apex-virtual-master' ##################################### # jobs ##################################### @@ -53,7 +53,17 @@ max-total: 5 max-per-node: 1 option: 'project' - + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'apex-verify.*' + - 'apex-.*-promote.*' + - 'apex-virtual.*' + - 'odl-netvirt-verify-virtual-create-apex-vms-.*' + - 'odl-netvirt-verify-virtual-install-netvirt-.*' + - 'functest-netvirt-virtual-suite-.*' + - 'odl-netvirt-verify-virtual-postprocess-.*' + block-level: 'NODE' scm: - git: url: https://gerrit.opnfv.org/gerrit/apex @@ -69,7 +79,7 @@ - string: name: NETVIRT_ARTIFACT default: distribution-karaf.tar.gz - - 'odl-netvirt-virtual-intel-defaults' + - 'apex-virtual-master-defaults' triggers: - gerrit: @@ -136,7 +146,7 @@ projects: - name: 'functest-netvirt-virtual-suite-master' predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l3-nofeature-ha + DEPLOY_SCENARIO=os-odl-nofeature-ha FUNCTEST_MODE=testcase FUNCTEST_SUITE_NAME=odl_netvirt RC_FILE_PATH=$HOME/cloner-info/overcloudrc @@ -206,7 +216,7 @@ - '{installer}-defaults' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-bgpvpn-noha' + default: 'os-odl-nofeature-noha' description: 'Scenario to deploy and test' - string: name: GS_URL diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index dfaf2a8b5..f5c6ab108 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -156,16 +156,6 @@ fi # start deployment sudo ${DEPLOY_CMD} -d ${DEPLOY_FILE} -n ${NETWORK_FILE} --debug -if [[ "$PROMOTE" == 'True' ]]; then - echo "CSIT job: setting host route for floating ip routing" - # csit route to allow docker container to reach floating ips - UNDERCLOUD=$(sudo virsh domifaddr undercloud | grep -Eo "[0-9\.]+{3}[0-9]+") - if sudo route | grep 192.168.37.128 > /dev/null; then - sudo route del -net 192.168.37.128 netmask 255.255.255.128 - fi - sudo route add -net 192.168.37.128 netmask 255.255.255.128 gw ${UNDERCLOUD} -fi - echo echo "--------------------------------------------------------" echo "Done!" diff --git a/jjb/apex/apex-snapshot-create.sh b/jjb/apex/apex-snapshot-create.sh index 7e38d56b4..b73931470 100644 --- a/jjb/apex/apex-snapshot-create.sh +++ b/jjb/apex/apex-snapshot-create.sh @@ -91,10 +91,16 @@ echo "Snapshot saved as apex-${SNAP_TYPE}-snap-${DATE}.tar.gz" # update opnfv properties file if [ "$SNAP_TYPE" == 'csit' ]; then - curl -O -L http://$GS_URL/snapshot.properties - sed -i '/^OPNFV_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#};${x;/^$/{s##OPNFV_SNAP_URL='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties snap_sha=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1) - sed -i '/^OPNFV_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//OPNFV_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties + if curl --fail -O -L http://$GS_URL/snapshot.properties; then + sed -i '/^OPNFV_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#};${x;/^$/{s##OPNFV_SNAP_URL='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties + sed -i '/^OPNFV_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//OPNFV_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties + else + cat << EOF > snapshot.properties +OPNFV_SNAP_URL=${GS_URL}/apex-csit-snap-${DATE}.tar.gz +OPNFV_SNAP_SHA512SUM=${snap_sha} +EOF + fi echo "OPNFV_SNAP_URL=$GS_URL/apex-csit-snap-${DATE}.tar.gz" echo "OPNFV_SNAP_SHA512SUM=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1)" echo "Updated properties file: " diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh index a93421c8e..1810a0036 100644 --- a/jjb/apex/apex-snapshot-deploy.sh +++ b/jjb/apex/apex-snapshot-deploy.sh @@ -141,14 +141,15 @@ done mkdir -p $HOME/cloner-info cp -f overcloudrc $HOME/cloner-info/ -admin_controller_ip=$(cat overcloudrc | grep -Eo -m 1 "192.0.2.[0-9]+") +admin_controller_ip=$(cat overcloudrc | grep -Eo -m 1 "192.0.2.[0-9]+" | head -1) netvirt_url="http://${admin_controller_ip}:8081/restconf/operational/network-topology:network-topology/topology/netvirt:1" source overcloudrc counter=1 while [ "$counter" -le 10 ]; do - if curl --fail --silent ${admin_controller_ip}:80 > /dev/null; then - echo "Overcloud Horizon is up...Checking if OpenDaylight NetVirt is up..." + echo "Checking if OpenStack is up" + if nc -z ${admin_controller_ip} 9696 > /dev/null; then + echo "Overcloud Neutron is up...Checking if OpenDaylight NetVirt is up..." if curl --fail --silent -u admin:admin ${netvirt_url} > /dev/null; then echo "OpenDaylight is up. Overcloud deployment complete" exit 0 @@ -156,7 +157,7 @@ while [ "$counter" -le 10 ]; do echo "OpenDaylight not yet up, try ${counter}" fi else - echo "Horizon/Apache not yet up, try ${counter}" + echo "Neutron not yet up, try ${counter}" fi counter=$((counter+1)) sleep 60 diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index 8743368fb..5c777a824 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -24,7 +24,9 @@ importkey () { git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null #this is where we import the siging key if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then - source $WORKSPACE/releng/utils/gpg_import_key.sh + if ! $WORKSPACE/releng/utils/gpg_import_key.sh; then + echo "WARNING: Failed to run gpg key import" + fi fi } @@ -75,13 +77,16 @@ uploadrpm () { uploadsnap () { # Uploads snapshot artifact and updated properties file echo "Uploading snapshot artifacts" + # snapshot dir is the same node in the create job workspace + # only 1 promotion job can run at a time on a slave + snapshot_dir="${WORKSPACE}/../apex-create-snapshot" if [ -z "$SNAP_TYPE" ]; then echo "ERROR: SNAP_TYPE not provided...exiting" exit 1 fi - gsutil cp $WORKSPACE/apex-${SNAP_TYPE}-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log + gsutil cp ${snapshot_dir}/apex-${SNAP_TYPE}-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log if [ "$SNAP_TYPE" == 'csit' ]; then - gsutil cp $WORKSPACE/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log + gsutil cp ${snapshot_dir}/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log fi echo "Upload complete for Snapshot" } diff --git a/jjb/apex/apex-verify-jobs.yaml b/jjb/apex/apex-verify-jobs.yaml index 2f29765d2..81e59bdf5 100644 --- a/jjb/apex/apex-verify-jobs.yaml +++ b/jjb/apex/apex-verify-jobs.yaml @@ -312,6 +312,7 @@ - 'apex-verify.*' - 'apex-virtual.*' - 'apex-.*-promote.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -343,6 +344,14 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true + - shell: | + if echo $DEPLOY_SCENARIO | grep queens &> /dev/null; then + echo 'REPO=opnfv' > functest_repo + else + echo 'REPO=ollivier' > functest_repo + fi + - inject: + properties-file: functest_repo - multijob: name: functest-smoke condition: ALWAYS @@ -353,6 +362,7 @@ DEPLOY_SCENARIO=$DEPLOY_SCENARIO FUNCTEST_SUITE_NAME=tempest_smoke_serial FUNCTEST_MODE=testcase + REPO=$REPO GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER diff --git a/jjb/apex/apex.yaml b/jjb/apex/apex.yaml index bb5b53d6d..e24a2c358 100644 --- a/jjb/apex/apex.yaml +++ b/jjb/apex/apex.yaml @@ -8,7 +8,7 @@ - 'apex-virtual-{stream}' - 'apex-deploy-{platform}-{stream}' - 'apex-daily-{stream}' - - 'apex-csit-promote-daily-{stream}-{os_version}' + - 'apex-csit-promote-daily-{stream}-os-{os_version}' - 'apex-fdio-promote-daily-{stream}' - 'apex-{scenario}-baremetal-{scenario_stream}' - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' @@ -144,10 +144,18 @@ <<: *danube - 'os-ovn-nofeature-noha': <<: *danube + - 'os-nosdn-nofeature-noha': + <<: *master + - 'os-nosdn-nofeature-ha': + <<: *master - 'os-odl-nofeature-noha': <<: *master - 'os-odl-nofeature-ha': <<: *master + - 'os-nosdn-queens-noha': + <<: *master + - 'os-nosdn-queens-ha': + <<: *master - 'os-odl-queens-noha': <<: *master - 'os-odl-queens-ha': @@ -339,6 +347,7 @@ - 'yardstick.*' - 'dovetail.*' - 'storperf.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -393,6 +402,7 @@ - 'apex-run.*' - 'apex-virtual-.*' - 'apex-verify-gate-.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -420,6 +430,14 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true + - shell: | + if echo $DEPLOY_SCENARIO | grep queens &> /dev/null; then + echo 'REPO=opnfv' > functest_repo + else + echo 'REPO=ollivier' > functest_repo + fi + - inject: + properties-file: functest_repo - multijob: name: functest-smoke condition: ALWAYS @@ -430,6 +448,7 @@ DEPLOY_SCENARIO=$DEPLOY_SCENARIO FUNCTEST_MODE=$FUNCTEST_MODE FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + REPO=$REPO GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -623,8 +642,7 @@ predefined-parameters: DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ - && $BUILD_NUMBER % 2 == 1" + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/" abort-all-job: false git-revision: false - multijob: @@ -637,8 +655,7 @@ predefined-parameters: DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ - && $BUILD_NUMBER % 2 == 0" + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/" abort-all-job: false git-revision: false - multijob: @@ -737,13 +754,11 @@ - job-template: name: 'apex-create-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - shell: !include-raw-escape: ./apex-snapshot-create.sh @@ -752,13 +767,11 @@ - job-template: name: 'apex-upload-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - inject: properties-content: ARTIFACT_TYPE=snapshot @@ -766,7 +779,7 @@ # CSIT promote - job-template: - name: 'apex-csit-promote-daily-{stream}-{os_version}' + name: 'apex-csit-promote-daily-{stream}-os-{os_version}' # Job template for promoting CSIT Snapshots # @@ -832,7 +845,7 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true @@ -844,7 +857,7 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true @@ -1493,6 +1506,22 @@ name: Baremetal Deploy and Test Phase condition: SUCCESSFUL projects: + - name: 'apex-os-nosdn-nofeature-noha-baremetal-master' + node-parameters: false + current-parameters: false + predefined-parameters: | + OPNFV_CLEAN=yes + kill-phase-on: NEVER + abort-all-job: true + git-revision: false + - name: 'apex-os-nosdn-nofeature-ha-baremetal-master' + node-parameters: false + current-parameters: false + predefined-parameters: | + OPNFV_CLEAN=yes + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - name: 'apex-os-odl-nofeature-noha-baremetal-master' node-parameters: false current-parameters: false @@ -1509,6 +1538,22 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false + - name: 'apex-os-nosdn-queens-noha-baremetal-master' + node-parameters: false + current-parameters: false + predefined-parameters: | + OPNFV_CLEAN=yes + kill-phase-on: NEVER + abort-all-job: true + git-revision: false + - name: 'apex-os-nosdn-queens-ha-baremetal-master' + node-parameters: false + current-parameters: false + predefined-parameters: | + OPNFV_CLEAN=yes + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - name: 'apex-os-odl-queens-noha-baremetal-master' node-parameters: false current-parameters: false diff --git a/jjb/apex/apex.yaml.j2 b/jjb/apex/apex.yaml.j2 index 3397f1ed2..1f5232b5f 100644 --- a/jjb/apex/apex.yaml.j2 +++ b/jjb/apex/apex.yaml.j2 @@ -8,7 +8,7 @@ - 'apex-virtual-{stream}' - 'apex-deploy-{platform}-{stream}' - 'apex-daily-{stream}' - - 'apex-csit-promote-daily-{stream}-{os_version}' + - 'apex-csit-promote-daily-{stream}-os-{os_version}' - 'apex-fdio-promote-daily-{stream}' - 'apex-{scenario}-baremetal-{scenario_stream}' - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' @@ -219,6 +219,7 @@ - 'yardstick.*' - 'dovetail.*' - 'storperf.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -273,6 +274,7 @@ - 'apex-run.*' - 'apex-virtual-.*' - 'apex-verify-gate-.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -300,6 +302,14 @@ kill-phase-on: FAILURE abort-all-job: true git-revision: true + - shell: | + if echo $DEPLOY_SCENARIO | grep queens &> /dev/null; then + echo 'REPO=opnfv' > functest_repo + else + echo 'REPO=ollivier' > functest_repo + fi + - inject: + properties-file: functest_repo - multijob: name: functest-smoke condition: ALWAYS @@ -310,6 +320,7 @@ DEPLOY_SCENARIO=$DEPLOY_SCENARIO FUNCTEST_MODE=$FUNCTEST_MODE FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + REPO=$REPO GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -503,8 +514,7 @@ predefined-parameters: DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ - && $BUILD_NUMBER % 2 == 1" + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/" abort-all-job: false git-revision: false - multijob: @@ -517,8 +527,7 @@ predefined-parameters: DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/ - && $BUILD_NUMBER % 2 == 0" + enable-condition: "def m = '$DEPLOY_SCENARIO' ==~ /os-(nosdn-nofeature|odl-bgpvpn)-ha/" abort-all-job: false git-revision: false - multijob: @@ -617,13 +626,11 @@ - job-template: name: 'apex-create-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - shell: !include-raw-escape: ./apex-snapshot-create.sh @@ -632,13 +639,11 @@ - job-template: name: 'apex-upload-snapshot' - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - disabled: false + parameters: + - '{project}-defaults' + builders: - inject: properties-content: ARTIFACT_TYPE=snapshot @@ -646,7 +651,7 @@ # CSIT promote - job-template: - name: 'apex-csit-promote-daily-{stream}-{os_version}' + name: 'apex-csit-promote-daily-{stream}-os-{os_version}' # Job template for promoting CSIT Snapshots # @@ -712,7 +717,7 @@ condition: SUCCESSFUL projects: - name: 'apex-create-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true @@ -724,7 +729,7 @@ condition: SUCCESSFUL projects: - name: 'apex-upload-snapshot' - current-parameters: false + current-parameters: true predefined-parameters: | SNAP_TYPE=csit node-parameters: true diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden index 8375f6163..71a6c3dbe 100644 --- a/jjb/apex/scenarios.yaml.hidden +++ b/jjb/apex/scenarios.yaml.hidden @@ -1,6 +1,10 @@ master: + - 'os-nosdn-nofeature-noha' + - 'os-nosdn-nofeature-ha' - 'os-odl-nofeature-noha' - 'os-odl-nofeature-ha' + - 'os-nosdn-queens-noha' + - 'os-nosdn-queens-ha' - 'os-odl-queens-noha' - 'os-odl-queens-ha' fraser: diff --git a/jjb/compass4nfv/compass-ci-jobs.yaml b/jjb/compass4nfv/compass-ci-jobs.yaml index 8eeaabfaa..298173e85 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yaml +++ b/jjb/compass4nfv/compass-ci-jobs.yaml @@ -222,12 +222,6 @@ - condition-kind: regex-match regex: os-(nosdn|odl_l3)-nofeature-ha label: '{scenario}' - - condition-kind: day-of-week - day-selector: select-days - days: - MON: true - TUES: true - use-build-time: true steps: - trigger-builds: - project: 'dovetail-compass-{pod}-default-{stream}' @@ -247,12 +241,6 @@ - condition-kind: regex-match regex: os-(nosdn|odl_l3)-nofeature-ha label: '{scenario}' - - condition-kind: day-of-week - day-selector: select-days - days: - THURS: true - FRI: true - use-build-time: true steps: - trigger-builds: - project: 'dovetail-compass-{pod}-proposed_tests-{stream}' diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index fe827808d..2a9c73c2f 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -60,6 +60,10 @@ releng_repo=${WORKSPACE}/releng [ -d ${releng_repo} ] && sudo rm -rf ${releng_repo} git clone https://gerrit.opnfv.org/gerrit/releng ${releng_repo} >/dev/null +pharos_repo=${WORKSPACE}/pharos +[ -d ${pharos_repo} ] && sudo rm -rf ${pharos_repo} +git clone https://git.opnfv.org/pharos ${pharos_repo} >/dev/null + if [[ ${INSTALLER_TYPE} != 'joid' ]]; then echo "SUT branch is $SUT_BRANCH" echo "dovetail branch is $BRANCH" @@ -93,10 +97,14 @@ if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "fuel" ]]; then fi cat $OPENRC +# These packages are used for parsing yaml files and decrypting ipmi user and password. +sudo pip install shyaml +sudo yum install -y rubygems || sudo apt-get install -y ruby +sudo gem install hiera-eyaml + if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then compass_repo=${WORKSPACE}/compass4nfv/ git clone https://github.com/opnfv/compass4nfv.git ${compass_repo} >/dev/null - sudo pip install shyaml scenario_file=${compass_repo}/deploy/conf/hardware_environment/$NODE_NAME/os-nosdn-nofeature-ha.yml ipmiIp=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiIp) ipmiPass=$(cat ${scenario_file} | shyaml get-value hosts.0.ipmiPass) @@ -120,11 +128,26 @@ if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_T fuel_ctl_ssh_options="${ssh_options} -i ${SSH_KEY}" ssh_user="ubuntu" fuel_ctl_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ - "sudo salt --out yaml 'ctl*' pillar.get _param:openstack_control_address | \ - awk '{print \$2; exit}'") &> /dev/null + "sudo salt 'cfg*' pillar.get _param:openstack_control_address --out text| \ + cut -f2 -d' '") + ipmi_index=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ + "sudo salt 'ctl*' network.ip_addrs cidr=${fuel_ctl_ip} --out text | grep ${fuel_ctl_ip} | cut -c 5") + organization="$(cut -d'-' -f1 <<< "${NODE_NAME}")" + pod_name="$(cut -d'-' -f2 <<< "${NODE_NAME}")" + pdf_file=${pharos_repo}/labs/${organization}/${pod_name}.yaml + ipmiIp=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.address) + ipmiIp="$(cut -d'/' -f1 <<< "${ipmiIp}")" + ipmiPass=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.pass) + ipmiUser=$(cat ${pdf_file} | shyaml get-value nodes.$[ipmi_index-1].remote_management.user) + [[ $ipmiUser == ENC* ]] && ipmiUser=$(eyaml decrypt -s ${ipmiUser//[[:blank:]]/}) + [[ $ipmiPass == ENC* ]] && ipmiPass=$(eyaml decrypt -s ${ipmiPass//[[:blank:]]/}) + cat << EOF >${DOVETAIL_CONFIG}/pod.yaml nodes: -- {ip: ${fuel_ctl_ip}, name: node1, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}} +- {ip: ${INSTALLER_IP}, name: node0, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, + role: Jumpserver, user: ${ssh_user}} +- {ip: ${fuel_ctl_ip}, name: node1, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, + role: controller, user: ${ssh_user}, ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}} EOF fi @@ -172,7 +195,7 @@ if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then if [ "$INSTALLER_TYPE" == "apex" ]; then attack_process='rabbitmq_server' else - attach_process='rabbitmq' + attack_process='rabbitmq' fi cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml process_info: diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index 24a28c69f..cf63bb5a4 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -13,7 +13,7 @@ check_os_deployment() { FUNCTEST_IMAGE=${REPO}/functest-healthcheck:${DOCKER_TAG} echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null - cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} check_deployment" + cmd="docker run --rm ${volumes} ${FUNCTEST_IMAGE} check_deployment" echo "Checking deployment, CMD: ${cmd}" eval ${cmd} ret_value=$? @@ -38,7 +38,7 @@ run_tiers() { FUNCTEST_IMAGE=${REPO}/functest-${tier}:${DOCKER_TAG} echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null - cmd="docker run --rm --privileged=true ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" + cmd="docker run --rm ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest tier '${tier}'. CMD: ${cmd}" eval ${cmd} ret_value=$? @@ -61,13 +61,13 @@ run_test() { case ${test_name} in connection_check|api_check|snaps_health_check) FUNCTEST_IMAGE=${REPO}/functest-healthcheck:${DOCKER_TAG} ;; - vping_ssh|vping_userdata|tempest_smoke_serial|rally_sanity|refstack_defcore|odl|odl_netvirt|snaps_smoke) + vping_ssh|vping_userdata|cinder_test|tempest_smoke_serial|rally_sanity|refstack_defcore|patrole|odl|snaps_smoke|neutron_trunk) FUNCTEST_IMAGE=${REPO}/functest-smoke:${DOCKER_TAG} ;; tempest_full_parallel|rally_full) FUNCTEST_IMAGE=${REPO}/functest-components:${DOCKER_TAG} ;; - cloudify_ims|orchestra_openims|orchestra_clearwaterims|vyos_vrouter) + cloudify_ims|vyos_vrouter|juju_epc) FUNCTEST_IMAGE=${REPO}/functest-vnf:${DOCKER_TAG} ;; - promise|doctor-notification|bgpvpn|functest-odl-sfc|domino-multinode|barometercollectd|fds) + doctor-notification|bgpvpn|functest-odl-sfc|barometercollectd|fds) FUNCTEST_IMAGE=${REPO}/functest-features:${DOCKER_TAG} ;; parser-basics) FUNCTEST_IMAGE=${REPO}/functest-parser:${DOCKER_TAG} ;; @@ -78,7 +78,7 @@ run_test() { esac echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null - cmd="docker run --rm --privileged=true ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" + cmd="docker run --rm ${envs} ${volumes} ${TESTCASE_OPTIONS} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'" echo "Running Functest test case '${test_name}'. CMD: ${cmd}" eval ${cmd} ret_value=$? @@ -169,9 +169,17 @@ elif [ ${FUNCTEST_MODE} == 'tier' ]; then run_tiers ${tiers} else if [ ${DEPLOY_TYPE} == 'baremetal' ] && [ "${HOST_ARCH}" != "aarch64" ]; then - tiers=(healthcheck smoke features vnf parser) + if [[ ${BRANCH} == "stable/fraser" ]]; then + tiers=(healthcheck smoke features vnf parser) + else + tiers=(healthcheck smoke features vnf) + fi else - tiers=(healthcheck smoke features parser) + if [[ ${BRANCH} == "stable/fraser" ]]; then + tiers=(healthcheck smoke features parser) + else + tiers=(healthcheck smoke features) + fi fi run_tiers ${tiers} fi diff --git a/jjb/functest/functest-daily-jobs.yaml b/jjb/functest/functest-daily-jobs.yaml index c81046c13..0300b99b2 100644 --- a/jjb/functest/functest-daily-jobs.yaml +++ b/jjb/functest/functest-daily-jobs.yaml @@ -237,7 +237,7 @@ - 'functest-{testsuite}-parameter' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-nofeature-ha' + default: 'os-nosdn-nofeature-noha' - string: name: CLEAN_DOCKER_IMAGES default: 'false' @@ -281,25 +281,27 @@ - 'connection_check' - 'api_check' - 'snaps_health_check' - - 'vping_userdata' - 'vping_ssh' + - 'vping_userdata' + - 'cinder_test' - 'tempest_smoke_serial' - 'rally_sanity' + - 'refstack_defcore' + - 'patrole' - 'odl' - - 'odl_netvirt' - 'snaps_smoke' - - 'refstack_defcore' - - 'promise' - - 'doctor' + - 'neutron_trunk' + - 'doctor-notification' - 'bgpvpn' - - 'parser' - - 'security_scan' + - 'functest-odl-sfc' + - 'barometercollectd' + - 'fds' - 'tempest_full_parallel' - 'rally_full' - 'cloudify_ims' - - 'cloudify_vrouter' - - 'orchestra_openims' - - 'orchestra_clearwaterims' + - 'vyos_vrouter' + - 'juju_epc' + - 'parser' default: 'connection_check' - choice: name: FUNCTEST_TIER @@ -309,6 +311,7 @@ - 'features' - 'components' - 'vnf' + - 'parser' default: 'healthcheck' - string: name: TESTCASE_OPTIONS diff --git a/jjb/functest/functest-docker.yaml b/jjb/functest/functest-docker.yaml index 50c7c3816..74738f280 100644 --- a/jjb/functest/functest-docker.yaml +++ b/jjb/functest/functest-docker.yaml @@ -25,6 +25,7 @@ # yamllint disable rule:key-duplicates image: - 'core' + - 'tempest' - 'healthcheck' - 'features' - 'components' @@ -32,6 +33,12 @@ - 'smoke' - 'vnf' + exclude: + - stream: 'master' + image: 'parser' + - stream: 'fraser' + image: 'tempest' + # settings for jobs run in multijob phases build-job-settings: &build-job-settings current-parameters: false @@ -107,6 +114,20 @@ - name: 'functest-core-docker-manifest-{stream}' <<: *manifest-job-settings - multijob: + name: 'build functest-tempest images' + execution-type: PARALLEL + projects: + - name: 'functest-tempest-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-tempest-docker-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish functest-tempest manifests' + execution-type: PARALLEL + projects: + - name: 'functest-tempest-docker-manifest-{stream}' + <<: *manifest-job-settings + - multijob: name: 'build all functest images' condition: SUCCESSFUL execution-type: PARALLEL diff --git a/jjb/releng/releng-ci-jobs.yaml b/jjb/releng/releng-ci-jobs.yaml index f9a2bc232..790bd7eb4 100644 --- a/jjb/releng/releng-ci-jobs.yaml +++ b/jjb/releng/releng-ci-jobs.yaml @@ -11,5 +11,9 @@ jjb-version: '2.0.3' lftools-version: '<1.0.0' + rtd-build-url: 'https://readthedocs.org/api/v2/webhook/opnfv-releng/38594/' + rtd-token: '291c6a0109493b4457e566d06141212452c65784' + jobs: - '{project-name}-ci-jobs' + - '{project-name}-rtd-jobs' diff --git a/jjb/sfc/sfc-project-jobs.yaml b/jjb/sfc/sfc-project-jobs.yaml index fea792467..70fc332b8 100644 --- a/jjb/sfc/sfc-project-jobs.yaml +++ b/jjb/sfc/sfc-project-jobs.yaml @@ -60,14 +60,43 @@ pattern: 'docs/**|.gitignore' builders: - - sfc-unit-tests + - sfc-unit-tests-and-docs + + publishers: + - sfc-unit-tests-and-docs-publisher ################################ # job builders ################################ - builder: - name: sfc-unit-tests + name: sfc-unit-tests-and-docs builders: - shell: | - cd $WORKSPACE && yamllint $(git ls-tree -r HEAD --name-only | egrep 'yml$|yaml$' | egrep -v 'scenarios') + cd $WORKSPACE && tox + +################################ +# job publishers +################################ +- publisher: + name: 'sfc-unit-tests-and-docs-publisher' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + fail-no-reports: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 0 + unhealthy: 0 + failing: 0 + - method: + healthy: 0 + unhealthy: 0 + failing: 0 + - email-jenkins-admins-on-failure diff --git a/releases/scripts/release-status.sh b/releases/scripts/release-status.sh index da66bc936..6790100a7 100644..100755 --- a/releases/scripts/release-status.sh +++ b/releases/scripts/release-status.sh @@ -12,6 +12,8 @@ set -o pipefail TAG="${TAG:-opnfv-6.0.0}" RELEASE="${RELEASE:-fraser}" +[ -a repos.txt ] && rm repos.txt + for project in releases/$RELEASE/*; do python releases/scripts/repos.py -n -f $project >> repos.txt done @@ -21,5 +23,3 @@ do tag="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/tags/$TAG")" echo "$repo $tag" done < repos.txt - -# rm repos.txt diff --git a/releases/scripts/repos.py b/releases/scripts/repos.py index 0ded0207b..58454c318 100644 --- a/releases/scripts/repos.py +++ b/releases/scripts/repos.py @@ -15,6 +15,38 @@ import argparse import yaml +class Repo(object): + """Object representing a repo listed in the release file. + + Includes eq, hash, and ne methods so set comparisons work + """ + + def __init__(self, repo=None, ref=None, version=None): + self.repo = repo + self.ref = ref + self.version = version + + def __repr__(self): + if self.version: + return "%s %s %s" % (self.repo, self.ref, self.version) + elif self.ref: + return "%s %s" % (self.repo, self.ref) + return "%s" % self.repo + + def __eq__(self, obj): + if isinstance(obj, Repo): + return ((self.repo == obj.repo) and + (self.ref == obj.ref) and + (self.version == obj.version)) + return False + + def __ne__(self, obj): + return (not self.__eq__(obj)) + + def __hash__(self): + return hash(self.__repr__()) + + def main(): """Given a release yamlfile list the repos it contains""" @@ -45,15 +77,17 @@ def list_repos(project, args): if 'releases' not in project: exit(0) + repos = set() for item in lookup: repo, ref = next(iter(item['location'].items())) if args.names: - print(repo) + repos.add(Repo(repo)) elif args.release and item['version'] == args.release: - print("%s %s" % (repo, ref)) + repos.add(Repo(repo, ref)) elif not args.release: - # Print all releases - print("%s %s %s" % (repo, item['version'], ref)) + repos.add(Repo(repo, ref, item['version'])) + for repo in repos: + print repo if __name__ == "__main__": @@ -15,9 +15,9 @@ setenv= [testenv:jjb] deps = - jenkins-job-builder==1.6.1 + jenkins-job-builder==2.0.3 commands= - jenkins-jobs test -o job_output -r jjb/global:{posargs:"jjb/"} + jenkins-jobs test -o job_output -r {posargs:"jjb/"} [testenv:docs] deps = -r{toxinidir}/docs/requirements.txt diff --git a/utils/build-server-ansible/vars/CentOS.yml b/utils/build-server-ansible/vars/CentOS.yml index 528388f27..7a2c1d1d5 100644 --- a/utils/build-server-ansible/vars/CentOS.yml +++ b/utils/build-server-ansible/vars/CentOS.yml @@ -46,6 +46,7 @@ - sysstat - xmlstarlet - facter + - jq - name: install the 'Development tools' package group yum: diff --git a/utils/build-server-ansible/vars/Ubuntu.yml b/utils/build-server-ansible/vars/Ubuntu.yml index e91c48569..f50255868 100644 --- a/utils/build-server-ansible/vars/Ubuntu.yml +++ b/utils/build-server-ansible/vars/Ubuntu.yml @@ -59,6 +59,7 @@ - sysstat - xmlstarlet - facter + - jq - name: Add Docker apt key. apt_key: diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index 4bc585885..b40b75b75 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -164,7 +164,13 @@ elif [ "$installer_type" == "apex" ]; then if [ -f /root/.ssh/id_rsa ]; then chmod 600 /root/.ssh/id_rsa fi - sudo scp $ssh_options root@$installer_ip:/home/stack/overcloudrc.v3 $dest_path + + if [ "${BRANCH}" == "stable/fraser" ]; then + rc_file=overcloudrc.v3 + else + rc_file=overcloudrc + fi + sudo scp $ssh_options root@$installer_ip:/home/stack/${rc_file} $dest_path elif [ "$installer_type" == "compass" ]; then if [ "${BRANCH}" == "stable/danube" ]; then diff --git a/utils/gpg_import_key.sh b/utils/gpg_import_key.sh index f45e40919..94ffbaab5 100755 --- a/utils/gpg_import_key.sh +++ b/utils/gpg_import_key.sh @@ -51,10 +51,14 @@ else if [ -z "$NODE_NAME" ]; then echo "Cannot find node name" exit 0 - else echo "Importing key for '$NODE_NAME'"; - gsutil cp gs://opnfv-signing-keys/"$NODE_NAME"-subkey . - gpg2 --import "$NODE_NAME"-subkey - rm -f "$NODE_NAME"-subkey - fi + elif gsutil ls gs://opnfv-signing-keys | grep $NODE_NAME; then + echo "Importing key for '$NODE_NAME'" + gsutil cp gs://opnfv-signing-keys/"$NODE_NAME"-subkey . + gpg2 --import "$NODE_NAME"-subkey + rm -f "$NODE_NAME"-subkey + else + echo "No keys found locally or remotely for host, skipping import" + exit 0 + fi fi |