diff options
Diffstat (limited to 'jjb')
| -rw-r--r-- | jjb/apex/apex.yml | 22 | ||||
| -rw-r--r-- | jjb/armband/armband-project-jobs.yml (renamed from jjb/armband/armband.yml) | 40 | ||||
| -rwxr-xr-x | jjb/armband/build.sh (renamed from jjb/armband/armband-build.sh) | 0 | ||||
| -rwxr-xr-x | jjb/armband/upload-artifacts.sh | 86 | ||||
| -rw-r--r-- | jjb/bottlenecks/bottlenecks-ci-jobs.yml | 40 | ||||
| -rw-r--r-- | jjb/functest/functest-ci-jobs.yml | 31 | ||||
| -rw-r--r-- | jjb/opnfv/opnfv-docker.sh | 8 | ||||
| -rw-r--r-- | jjb/opnfv/slave-params.yml | 15 | ||||
| -rw-r--r-- | jjb/releng-macros.yaml | 23 |
9 files changed, 220 insertions, 45 deletions
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 91f76b5cf..6dbea1a62 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -760,7 +760,7 @@ set -o pipefail # delete everything that is in $WORKSPACE - /bin/rm -rf $WORKSPACE + sudo /bin/rm -rf $WORKSPACE - builder: name: 'apex-upload-artifact' @@ -895,6 +895,26 @@ DEPLOY_CMD="./deploy.sh -c ${WORKSPACE}/build -r ${WORKSPACE}/build/images/" DEPLOY_FILE="${WORKSPACE}/config/deploy/${DEPLOY_SCENARIO}.yaml" NETWORK_FILE="${WORKSPACE}/config/network/network_settings.yaml" + # Make sure python34 is installed + if ! rpm -q python34 > /dev/null; then + sudo yum install -y epel-release + if ! sudo yum install -y python34; then + echo "Failed to install python34" + exit 1 + fi + fi + if ! rpm -q python34-PyYAML > /dev/null; then + sudo yum install -y epel-release + if ! sudo yum install -y python34-PyYAML; then + echo "Failed to install python34-PyYAML" + exit 1 + fi + fi + if [ -z ${PYTHONPATH:-} ]; then + export PYTHONPATH=${WORKSPACE}/lib/python + else + export PYTHONPATH=$PYTHONPATH:${WORKSPACE}/lib/python + fi else RPM_LIST=$RPM_INSTALL_PATH for pkg in common undercloud opendaylight-sfc; do diff --git a/jjb/armband/armband.yml b/jjb/armband/armband-project-jobs.yml index 75c0e7a7b..b40949d27 100644 --- a/jjb/armband/armband.yml +++ b/jjb/armband/armband-project-jobs.yml @@ -9,6 +9,7 @@ jobs: - 'armband-verify-{stream}' + - 'armband-build-daily-{stream}' stream: - master: @@ -72,15 +73,11 @@ max-per-node: 1 parameters: - - project-parameters: + - project-parameter: project: '{project}' - 'arm-build1-defaults' - - choice: - name: FORCE_BUILD - choices: - - 'false' - - 'true' - description: 'Force build even if there are no changes in the armband repo. Default false' + - armband-project-parameter: + gs-pathname: '{gs-pathname}' scm: - git-scm: @@ -89,18 +86,41 @@ branch: '{branch}' triggers: - -pollscm: + - pollscm: cron: '0 H/4 * * *' wrappers: - -timeout: + - timeout: timeout: 360 fail: true builders: - shell: - !include-raw-escape: ./armband-build.sh + !include-raw-escape: ./build.sh + - shell: + !include-raw-escape: ./upload-artifacts.sh publishers: - email: recipients: josep.puigdemont@enea.com armband@enea.com + +######################## +# parameter macros +######################## +- parameter: + name: armband-project-parameter + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." + - choice: + name: FORCE_BUILD + choices: + - 'false' + - 'true' + description: 'Force build even if there are no changes in the armband repo. Default false' diff --git a/jjb/armband/armband-build.sh b/jjb/armband/build.sh index 81917f6de..81917f6de 100755 --- a/jjb/armband/armband-build.sh +++ b/jjb/armband/build.sh diff --git a/jjb/armband/upload-artifacts.sh b/jjb/armband/upload-artifacts.sh new file mode 100755 index 000000000..f4e84e9d2 --- /dev/null +++ b/jjb/armband/upload-artifacts.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o pipefail + +# check if we built something +if [ -f $WORKSPACE/.noupload ]; then + echo "Nothing new to upload. Exiting." + /bin/rm -f $WORKSPACE/.noupload + exit 0 +fi + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +# storing ISOs for verify & merge jobs will be done once we get the disk array +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + # store ISO locally on NFS first + ISOSTORE="/home/jenkins/opnfv/iso_store" + if [[ -d "$ISOSTORE" ]]; then + # remove all but most recent 3 ISOs first to keep iso_mount clean & tidy + cd $ISOSTORE + ls -tp | grep -v '/' | tail -n +4 | xargs -d '\n' /bin/rm -f -- + + # store ISO + echo "Storing latest ISO in local storage" + touch .storing + /bin/cp -f $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso \ + $ISOSTORE/opnfv-$OPNFV_ARTIFACT_VERSION.iso + rm .storing + fi +fi + +# log info to console +echo "Uploading armband artifacts. This could take some time..." +echo + +echo "Started at $(date)" +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +elif [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Uploaded Armband Fuel ISO for a merged change" +fi +echo "Ended at $(date)" + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.iso.log on the machine where this build is done." + exit 1 +fi + +echo "Done!" +echo +echo "--------------------------------------------------------" +echo +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" +echo +echo "--------------------------------------------------------" +echo diff --git a/jjb/bottlenecks/bottlenecks-ci-jobs.yml b/jjb/bottlenecks/bottlenecks-ci-jobs.yml index cd1d89a5c..9d881233d 100644 --- a/jjb/bottlenecks/bottlenecks-ci-jobs.yml +++ b/jjb/bottlenecks/bottlenecks-ci-jobs.yml @@ -114,7 +114,6 @@ builders: - 'bottlenecks-env-cleanup' - - 'bottlenecks-fetch-os-creds' - 'bottlenecks-run-suite' publishers: @@ -125,12 +124,6 @@ # builder macros ######################## - builder: - name: bottlenecks-fetch-os-creds - builders: - - shell: - !include-raw ../../utils/fetch_os_creds.sh - -- builder: name: bottlenecks-env-cleanup builders: - shell: | @@ -162,21 +155,36 @@ set -e [[ $GERRIT_REFSPEC_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" - echo "Bottlenecks: ${SUITE_NAME} running now..." - - # pull latest image - docker pull opnfv/bottlenecks:$DOCKER_TAG >$redirect + echo "Bottlenecks: to pull image opnfv/bottlenecks:${DOCKER_TAG}" + docker pull opnfv/bottlenecks:$DOCKER_TAG >${redirect} - # run tests by using docker - opts="--privileged=true --rm" + echo "Bottlenecks: docker start running" + opts="--privileged=true -id" envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ -e NODE_NAME=${NODE_NAME} -e EXTERNAL_NET=${EXTERNAL_NETWORK} \ -e BOTTLENECKS_BRANCH=${BOTTLENECKS_BRANCH} -e GERRIT_REFSPEC_DEBUG=${GERRIT_REFSPEC_DEBUG} \ -e BOTTLENECKS_DB_TARGET=${BOTTLENECKS_DB_TARGET} -e PACKAGE_URL=${PACKAGE_URL}" + cmd="sudo docker run ${opts} ${envs} opnfv/bottlenecks:${DOCKER_TAG} /bin/bash" + echo "Bottlenecks: docker cmd running ${cmd}" + ${cmd} >${redirect} + + echo "Bottlenecks: obtain docker id" + container_id=$(docker ps | grep "opnfv/bottlenecks:${DOCKER_TAG}" | awk '{print $1}' | head -1) + if [ -z ${container_id} ]; then + echo "Cannot find opnfv/bottlenecks container ID ${container_id}. Please check if it exists." + docker ps -a + exit 1 + fi + + echo "Bottlenecks: to prepare openstack environment" + prepare_env="${REPO_DIR}/ci/prepare_env.sh" + echo "Bottlenecks: docker cmd running: ${prepare_env}" + sudo docker exec ${container_id} ${prepare_env} + + echo "Bottlenecks: to run testsuite ${SUITE_NAME}" run_testsuite="${REPO_DIR}/run_tests.sh -s ${SUITE_NAME}" - cmd="sudo docker run ${opts} ${envs} opnfv/bottlenecks ${run_testsuite}" - echo "Bottlenecks: docker cmd running: ${cmd}" - ${cmd} + echo "Bottlenecks: docker cmd running: ${run_testsuite}" + sudo docker exec ${container_id} ${run_testsuite} #################### # parameter macros diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml index d691140f4..a0a1326ba 100644 --- a/jjb/functest/functest-ci-jobs.yml +++ b/jjb/functest/functest-ci-jobs.yml @@ -94,6 +94,9 @@ - nokia-pod1: installer: apex <<: *master + - arm-pod1: + installer: fuel + <<: *master #-------------------------------- testsuite: @@ -146,8 +149,6 @@ branch: '{branch}' builders: - - 'functest-cleanup' - - 'set-functest-env' - 'functest-{testsuite}-builder' ######################## @@ -219,11 +220,9 @@ builders: - shell: | #!/bin/bash - set +e - flags="-s" - [[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+=" -r" + set -e echo "Functest: run $FUNCTEST_SUITE_NAME" - cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME ${flags}" + cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME" container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd @@ -244,15 +243,15 @@ builders: - shell: | #!/bin/bash - set +e + set -e [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" # labconfig is used only for joid labconfig="" if [[ ${INSTALLER_TYPE} == 'apex' ]]; then - if virsh list | grep instack; then + if sudo virsh list | grep instack; then instack_mac=$(sudo virsh domiflist instack | grep default | \ grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") - elif virsh list | grep undercloud; then + elif sudo virsh list | grep undercloud; then instack_mac=$(sudo virsh domiflist undercloud | grep default | \ grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") else @@ -261,8 +260,12 @@ fi INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'}) sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" - sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable - sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable + if sudo iptables -C FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then + sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable + fi + if sudo iptables -C FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then + sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable + fi elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then # If production lab then creds may be retrieved dynamically # creds are on the jumphost, always in the same folder @@ -286,7 +289,7 @@ cmd="sudo docker run --privileged=true -id ${envs} ${labconfig} ${sshkey} ${res_volume} opnfv/functest:${DOCKER_TAG} /bin/bash" echo "Functest: Running docker run command: ${cmd}" ${cmd} >${redirect} - docker ps -a + docker ps -a >${redirect} sleep 5 container_id=$(docker ps | grep "opnfv/functest:${DOCKER_TAG}" | awk '{print $1}' | head -1) echo "Container ID=${container_id}" @@ -298,7 +301,7 @@ echo "Starting the container: docker start ${container_id}" docker start ${container_id} sleep 5 - docker ps + docker ps >${redirect} if [ $(docker ps | grep "opnfv/functest:${DOCKER_TAG}" | wc -l) == 0 ]; then echo "The container opnfv/functest with ID=${container_id} has not been properly started. Exiting..." exit 1 @@ -330,7 +333,7 @@ # Remove existing images if exist if [[ ! -z $(docker images | grep opnfv/functest) ]]; then echo "Docker images to remove:" - docker images | head -1 && docker images | grep opnfv/functest + docker images | head -1 && docker images | grep opnfv/functest >${redirect} image_tags=($(docker images | grep opnfv/functest | awk '{print $2}')) for tag in "${image_tags[@]}"; do echo "Removing docker image opnfv/functest:$tag..." diff --git a/jjb/opnfv/opnfv-docker.sh b/jjb/opnfv/opnfv-docker.sh index 8f44268e6..702c0ce52 100644 --- a/jjb/opnfv/opnfv-docker.sh +++ b/jjb/opnfv/opnfv-docker.sh @@ -17,6 +17,14 @@ echo "--------------------------------------------------------" echo +if [[ -n $(ps -ef|grep 'docker build'|grep -v grep) ]]; then + echo "There is already another build process in progress:" + echo $(ps -ef|grep 'docker build'|grep -v grep) + # Abort this job since it will colide and might mess up the current one. + echo "Aborting..." + exit 1 +fi + # Remove previous running containers if exist if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then echo "Removing existing $DOCKER_REPO_NAME containers..." diff --git a/jjb/opnfv/slave-params.yml b/jjb/opnfv/slave-params.yml index 2431ade85..2ca4459dd 100644 --- a/jjb/opnfv/slave-params.yml +++ b/jjb/opnfv/slave-params.yml @@ -597,3 +597,18 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' + +- parameter: + name: 'arm-pod1-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - arm-pod1 + default-slaves: + - arm-pod1 + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index 23d174a26..7fb4d653d 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -339,11 +339,27 @@ # generate and upload lint log echo "Running flake8 code on $PROJECT ..." - echo -e "Flake8 Violations\n-----------------" >> lint.log - find . \ + # Get number of flake8 violations. If none, this will be an + # empty string: "" + FLAKE_COUNT="$(find . \ -path './releng_flake8' -prune -o \ -type f -name "*.py" -print | \ - xargs flake8 --exit-zero --first >> lint.log + xargs flake8 --exit-zero -qq --count 2>&1)" + + if [ ! -z $FLAKE_COUNT ]; then + echo "Flake8 Violations: $FLAKE_COUNT" >> lint.log + find . \ + -path './releng_flake8' -prune -o \ + -type f -name "*.py" -print | \ + xargs flake8 --exit-zero --first >> violation.log + SHOWN=$(wc -l violation.log | cut -d' ' -f1) + echo -e "First $SHOWN shown\n---" >> lint.log + cat violation.log >> lint.log + sed -r -i '4,$s/^/ /g' lint.log + rm violation.log + else + echo -e "Flake8 Violations: 0" > lint.log + fi deactivate @@ -357,7 +373,6 @@ set -o xtrace export PATH=$PATH:/usr/local/bin/ if [[ -e lint.log ]] ; then - sed -r -i 's/^/ /g' lint.log echo -e "\nposting linting report to gerrit...\n" cat lint.log echo |