diff options
21 files changed, 199 insertions, 431 deletions
diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh index f25325129..2d39e98b0 100755 --- a/jjb/apex/apex-download-artifact.sh +++ b/jjb/apex/apex-download-artifact.sh @@ -23,7 +23,7 @@ if [[ "$ARTIFACT_VERSION" =~ dev ]]; then tar -xvf apex-${OPNFV_ARTIFACT_VERSION}.tar.gz popd > /dev/null else - echo "Will download RPMs and ISO..." + echo "Will download RPMs..." # Must be RPMs/ISO export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d") @@ -56,15 +56,6 @@ else echo "Unable to install new RPMs: $RPM_LIST" exit 1 fi - - # log info to console - echo "Downloading the ISO artifact using URL http://$OPNFV_ARTIFACT_URL" - echo "--------------------------------------------------------" - echo - - # Download ISO - curl --fail -s -o $BUILD_DIRECTORY/apex.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 - fi # TODO: Uncomment these lines to verify SHA512SUMs once the sums are diff --git a/jjb/apex/apex-gs-cleanup.sh b/jjb/apex/apex-gs-cleanup.sh deleted file mode 100755 index 1629aa85e..000000000 --- a/jjb/apex/apex-gs-cleanup.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -# log info to console -echo "Cleaning Google Storage" -echo "-----------------------" -echo - -thirty_days_ago=$(date -d "30 days ago" +"%Y%m%d") - -for i in $(gsutil ls gs://$GS_URL/*201?*); do - filedate=$(date -d "$(echo $i | grep -Eo 201[0-9]-?[0-9][0-9]-?[0-9][0-9])" +"%Y%m%d") - if [ $filedate -lt $thirty_days_ago ]; then - # gsutil indicates what it is removing so no need for output here - gsutil rm $i - fi -done diff --git a/jjb/apex/apex-iso-verify.sh b/jjb/apex/apex-iso-verify.sh index cdeac04d7..f102421f3 100755 --- a/jjb/apex/apex-iso-verify.sh +++ b/jjb/apex/apex-iso-verify.sh @@ -8,8 +8,6 @@ echo "Starting the Apex iso verify." echo "--------------------------------------------------------" echo -BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY - source $BUILD_DIRECTORY/../opnfv.properties if ! rpm -q virt-install > /dev/null; then @@ -29,76 +27,37 @@ fi # Make sure a pre-existing iso-verify isn't there rm_apex_iso_verify +#make sure there is not an existing console log file for the VM +sudo rm -f /var/log/libvirt/qemu/apex-iso-verify-console.log + # run an install from the iso # This streams a serial console to tcp port 3737 on localhost sudo virt-install -n apex-iso-verify -r 4096 --vcpus 4 --os-variant=rhel7 \ - --accelerate -v --noautoconsole --nographics \ + --accelerate -v --noautoconsole \ --disk path=/var/lib/libvirt/images/apex-iso-verify.qcow2,size=30,format=qcow2 \ -l $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso \ --extra-args 'console=ttyS0 console=ttyS0,115200n8 serial inst.ks=file:/iso-verify.ks inst.stage2=hd:LABEL=OPNFV\x20CentOS\x207\x20x86_64:/' \ --initrd-inject $BUILD_DIRECTORY/../ci/iso-verify.ks \ - --serial tcp,host=:3737,protocol=raw - -# Attach to tcpport 3737 and echo the output to stdout -# watch for a 5 min time out, a power off message or a tcp disconnect -python << EOP -#!/usr/bin/env python - -import sys -import socket -from time import sleep -from time import time - - -TCP_IP = '127.0.0.1' -TCP_PORT = 3737 -BUFFER_SIZE = 1024 - -try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((TCP_IP, TCP_PORT)) -except Exception, e: - print "Failed to connect to the iso-verofy vm's serial console" - print "this probably means that the VM failed to start" - raise e - -activity = time() -data = s.recv(BUFFER_SIZE) -last_data = data -while time() - activity < 300: - try: - if data != last_data: - activity = time() - last_data = data - data = s.recv(BUFFER_SIZE) - sys.stdout.write(data) - if 'Powering off' in data: - break - sleep(.5) - except socket.error, e: - # for now assuming that the connection was closed - # which is good, means the vm finished installing - # printing the error output just in case we need to debug - print "VM console connection lost: %s" % msg - break -s.close() - -if time() - activity > 300: - print "failing due to console inactivity" - exit(1) -else: - print "Success!" -EOP - -# save the python return code for after cleanup -python_rc=$? + --serial file,path=/var/log/libvirt/qemu/apex-iso-verify-console.log + +echo "Waiting for install to finish..." +sleep 10 +end_time=$(($SECONDS+1500)) +while ! [[ `sudo tail -n1 /var/log/libvirt/qemu/apex-iso-verify-console.log` =~ 'Power down' ]]; do + if [ $SECONDS -gt $end_time ] || ! sudo virsh list --all | grep apex-iso-verify | grep running > /dev/null; then + sudo cat /var/log/libvirt/qemu/apex-iso-verify-console.log + sudo virsh list --all + echo "Error: Failed to find power down message after install" + exit 1 + fi + sleep 10 +done + +sudo cat /var/log/libvirt/qemu/apex-iso-verify-console.log # clean up rm_apex_iso_verify -# Exit with the RC of the Python job -exit $python_rc - echo echo "--------------------------------------------------------" echo "Done!" diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index de565ed66..ae2186002 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -13,7 +13,7 @@ - 'apex-csit-promote-daily-{stream}' - 'apex-fdio-promote-daily-{stream}' - 'apex-verify-iso-{stream}' - - 'apex-deploy-test-baremetal-{stream}' + - 'apex-run-deploy-test-baremetal-{stream}' - 'apex-upload-snapshot' - 'apex-create-snapshot' # stream: branch with - in place of / (eg. stable-arno) @@ -545,7 +545,7 @@ # Baremetal Deploy and Test - job-template: - name: 'apex-deploy-test-baremetal-{stream}' + name: 'apex-run-deploy-test-baremetal-{stream}' # Job template for daily build # @@ -578,9 +578,9 @@ block-level: 'NODE' blocking-jobs: - 'apex-verify.*' - - 'apex-deploy.*' - 'apex-runner.*' - 'apex-.*-promote.*' + - 'apex-run.*' builders: - description-setter: description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" @@ -593,6 +593,9 @@ current-parameters: true predefined-parameters: | OPNFV_CLEAN=yes + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC= + DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: FAILURE abort-all-job: true git-revision: false @@ -678,7 +681,7 @@ - name: 'apex-verify-iso-danube' current-parameters: false predefined-parameters: | - BUILD_DIRECTORY=apex-build-danube/.build + BUILD_DIRECTORY=$WORKSPACE/../apex-build-danube/.build GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC= GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -692,7 +695,7 @@ condition: SUCCESSFUL projects: - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -702,7 +705,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -712,7 +715,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -722,7 +725,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -732,7 +735,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -742,7 +745,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -752,7 +755,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -762,7 +765,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -772,7 +775,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -782,7 +785,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -792,7 +795,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -802,7 +805,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -812,7 +815,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -822,7 +825,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -832,7 +835,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -842,7 +845,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -852,7 +855,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -862,7 +865,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -872,7 +875,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -882,7 +885,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -892,7 +895,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -902,7 +905,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -912,7 +915,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -922,7 +925,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -932,7 +935,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -942,7 +945,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-danube' + - name: 'apex-run-deploy-test-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1009,7 +1012,7 @@ - name: 'apex-verify-iso-master' current-parameters: false predefined-parameters: | - BUILD_DIRECTORY=apex-build-master/.build + BUILD_DIRECTORY=$WORKSPACE/../apex-build-master/.build GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC= GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -1023,7 +1026,7 @@ condition: SUCCESSFUL projects: - - name: 'apex-deploy-test-baremetal-master' + - name: 'apex-run-deploy-test-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1033,7 +1036,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-master' + - name: 'apex-run-deploy-test-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1043,7 +1046,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-master' + - name: 'apex-run-deploy-test-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1053,7 +1056,7 @@ abort-all-job: true git-revision: false - - name: 'apex-deploy-test-baremetal-master' + - name: 'apex-run-deploy-test-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1290,30 +1293,6 @@ abort-all-job: true git-revision: false -- job-template: - name: 'apex-gs-clean-{stream}' - - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - node: '{slave}' - - disabled: false - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - builders: - - 'apex-gs-clean' - - triggers: - - 'apex-gs-clean-{stream}' - ######################## # parameter macros ######################## @@ -1390,12 +1369,6 @@ !include-raw: ./apex-download-artifact.sh - builder: - name: 'apex-gs-cleanup' - builders: - - shell: - !include-raw: ./apex-gs-cleanup.sh - -- builder: name: 'apex-deploy' builders: - shell: @@ -1412,7 +1385,3 @@ name: 'apex-danube' triggers: - timed: '0 12 * * *' -- trigger: - name: 'apex-gs-clean-{stream}' - triggers: - - timed: '0 2 * * *'
\ No newline at end of file diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index d739a7068..3f10e214b 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -13,7 +13,7 @@ - 'apex-csit-promote-daily-{stream}' - 'apex-fdio-promote-daily-{stream}' - 'apex-verify-iso-{stream}' - - 'apex-deploy-test-baremetal-{stream}' + - 'apex-run-deploy-test-baremetal-{stream}' - 'apex-upload-snapshot' - 'apex-create-snapshot' # stream: branch with - in place of / (eg. stable-arno) @@ -545,7 +545,7 @@ # Baremetal Deploy and Test - job-template: - name: 'apex-deploy-test-baremetal-{stream}' + name: 'apex-run-deploy-test-baremetal-{stream}' # Job template for daily build # @@ -578,9 +578,9 @@ block-level: 'NODE' blocking-jobs: - 'apex-verify.*' - - 'apex-deploy.*' - 'apex-runner.*' - 'apex-.*-promote.*' + - 'apex-run.*' builders: - description-setter: description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" @@ -593,6 +593,9 @@ current-parameters: true predefined-parameters: | OPNFV_CLEAN=yes + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC= + DEPLOY_SCENARIO=$DEPLOY_SCENARIO kill-phase-on: FAILURE abort-all-job: true git-revision: false @@ -678,7 +681,7 @@ - name: 'apex-verify-iso-{{ stream }}' current-parameters: false predefined-parameters: | - BUILD_DIRECTORY=apex-build-{{ stream }}/.build + BUILD_DIRECTORY=$WORKSPACE/../apex-build-{{ stream }}/.build GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC= GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER @@ -692,7 +695,7 @@ condition: SUCCESSFUL projects: {% for scenario in scenarios[stream] %} - - name: 'apex-deploy-test-baremetal-{{ stream }}' + - name: 'apex-run-deploy-test-baremetal-{{ stream }}' node-parameters: false current-parameters: false predefined-parameters: | @@ -929,30 +932,6 @@ abort-all-job: true git-revision: false -- job-template: - name: 'apex-gs-clean-{stream}' - - # Job template for clean - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - node: '{slave}' - - disabled: false - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - builders: - - 'apex-gs-clean' - - triggers: - - 'apex-gs-clean-{stream}' - ######################## # parameter macros ######################## @@ -1029,12 +1008,6 @@ !include-raw: ./apex-download-artifact.sh - builder: - name: 'apex-gs-cleanup' - builders: - - shell: - !include-raw: ./apex-gs-cleanup.sh - -- builder: name: 'apex-deploy' builders: - shell: @@ -1051,7 +1024,4 @@ name: 'apex-danube' triggers: - timed: '0 12 * * *' -- trigger: - name: 'apex-gs-clean-{stream}' - triggers: - - timed: '0 2 * * *' + diff --git a/jjb/barometer/barometer.yml b/jjb/barometer/barometer.yml index 9ec30e809..68b8a04c0 100644 --- a/jjb/barometer/barometer.yml +++ b/jjb/barometer/barometer.yml @@ -126,21 +126,16 @@ disabled: '{obj:disabled}' - concurrent: true + concurrent: false properties: - logrotate-default - - throttle: - enabled: true - max-total: 3 - max-per-node: 2 - option: 'project' parameters: - project-parameter: project: '{project}' branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - 'opnfv-build-centos-defaults' scm: - git-scm @@ -151,7 +146,6 @@ builders: - shell: | pwd - cd src - ./install_build_deps.sh - make clobber - make + cd ci + ./install_dependencies.sh + ./build_rpm.sh diff --git a/jjb/dovetail/dovetail-artifacts-upload.sh b/jjb/dovetail/dovetail-artifacts-upload.sh index b23decad1..f1a9e7222 100755 --- a/jjb/dovetail/dovetail-artifacts-upload.sh +++ b/jjb/dovetail/dovetail-artifacts-upload.sh @@ -52,7 +52,7 @@ echo "signature Upload Complete!" upload () { # log info to console -echo "Uploading to artifact. This could take some time..." +echo "Uploading ${STORE_FILE_NAME} to artifact. This could take some time..." echo cd $WORKSPACE diff --git a/jjb/dovetail/dovetail-artifacts-upload.yml b/jjb/dovetail/dovetail-artifacts-upload.yml index 3d9af5ed7..0c8efbe0d 100644 --- a/jjb/dovetail/dovetail-artifacts-upload.yml +++ b/jjb/dovetail/dovetail-artifacts-upload.yml @@ -19,6 +19,8 @@ - 'dovetail' - 'functest' - 'yardstick' + - 'testapi' + - 'mongo' ############################################# # job template @@ -55,7 +57,8 @@ builders: - 'dovetail-builder-artifacts-upload' - - 'dovetail-workspace-cleanup' + - 'dovetail-upload-artifacts-cache-cleanup' + - 'dovetail-images-cleanup' #################### # parameter macros @@ -94,7 +97,7 @@ !include-raw: ./dovetail-artifacts-upload.sh - builder: - name: dovetail-workspace-cleanup + name: dovetail-upload-artifacts-cache-cleanup builders: - shell: | #!/bin/bash @@ -104,27 +107,8 @@ /bin/rm -rf $CACHE_DIR - # Remove previous running containers if exist - if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then - echo "Removing existing $DOCKER_REPO_NAME containers..." - docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f - t=60 - # Wait max 60 sec for containers to be removed - while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do - sleep 1 - let t=t-1 - done - fi - - # Remove existing images if exist - if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep $DOCKER_REPO_NAME - image_tags=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $2}')) - for tag in "${image_tags[@]}"; do - if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $tag)" ]]; then - echo "Removing docker image $DOCKER_REPO_NAME:$tag..." - docker rmi -f $DOCKER_REPO_NAME:$tag - fi - done - fi +- builder: + name: dovetail-images-cleanup + builders: + - shell: + !include-raw: ./dovetail-cleanup.sh diff --git a/jjb/dovetail/dovetail-cleanup.sh b/jjb/dovetail/dovetail-cleanup.sh index 22b2ba2ce..0ee789a97 100755 --- a/jjb/dovetail/dovetail-cleanup.sh +++ b/jjb/dovetail/dovetail-cleanup.sh @@ -2,8 +2,8 @@ [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" -#clean up dependent project docker images, which has no containers and image tag None -clean_images=(opnfv/functest opnfv/yardstick) +# clean up dependent project docker images, which has no containers and image tag None +clean_images=(opnfv/functest opnfv/yardstick opnfv/testapi mongo) for clean_image in "${clean_images[@]}"; do echo "Removing image $image_id, which has no containers and image tag is None" dangling_images=($(docker images -f "dangling=true" | grep ${clean_image} | awk '{print $3}')) @@ -14,7 +14,7 @@ for clean_image in "${clean_images[@]}"; do fi done -echo "Remove containers with image dovetail:<None>..." +echo "Remove containers with image opnfv/dovetail:<None>..." dangling_images=($(docker images -f "dangling=true" | grep opnfv/dovetail | awk '{print $3}')) if [[ -n ${dangling_images} ]]; then for image_id in "${dangling_images[@]}"; do @@ -24,7 +24,7 @@ if [[ -n ${dangling_images} ]]; then done fi -echo "Cleaning up dovetail docker containers/images..." +echo "Cleaning up dovetail docker containers..." if [[ ! -z $(docker ps -a | grep opnfv/dovetail) ]]; then echo "Removing existing opnfv/dovetail containers..." docker ps -a | grep opnfv/dovetail | awk '{print $1}' | xargs docker rm -f >${redirect} diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 2fa868779..5432c9447 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -73,8 +73,8 @@ auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-odl_l2-sfc-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - # - 'os-odl_l2-bgpvpn-ha': - # auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' + - 'os-odl_l2-bgpvpn-ha': + auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-ovs-ha': @@ -112,7 +112,6 @@ jobs: - 'fuel-{scenario}-{pod}-daily-{stream}' - 'fuel-deploy-{pod}-daily-{stream}' - - 'fuel-os-odl_l2-bgpvpn-ha-{pod}-daily-{stream}' ######################## # job templates @@ -190,107 +189,32 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - - publishers: - - email: - recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com - -- job-template: - name: 'fuel-os-odl_l2-bgpvpn-ha-{pod}-daily-{stream}' - - disabled: '{obj:disabled}' - - concurrent: false - - properties: - - logrotate-default - - throttle: - enabled: true - max-total: 4 - max-per-node: 1 - option: 'project' - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'fuel-os-.*?-{pod}-daily-.*' - - 'fuel-os-.*?-{pod}-weekly-.*' - block-level: 'NODE' - - wrappers: - - build-name: - name: '$BUILD_NUMBER - Scenario: os-odl_l2-bgpvpn-ha' - - triggers: - - 'fuel-os-odl_l2-bgpvpn-ha-{pod}-daily-{stream}-trigger' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - '{installer}-defaults' - - '{slave-label}-defaults': - installer: '{installer}' - - string: - name: DEPLOY_SCENARIO - default: "os-odl_l2-bgpvpn-ha" - - fuel-ci-parameter: - gs-pathname: '{gs-pathname}' - - builders: - - description-setter: - description: "Built on $NODE_NAME" - - trigger-builds: - - project: 'fuel-deploy-{pod}-daily-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-bgpvpn-ha - same-node: true - block: true - - trigger-builds: - - project: 'functest-fuel-{pod}-daily-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-bgpvpn-ha - same-node: true - block: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-fuel-{pod}-daily-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-bgpvpn-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - # 1.dovetail only master by now, not sync with A/B/C branches + # 1.dovetail only has master, based on D release # 2.here the stream means the SUT stream, dovetail stream is defined in its own job - # 3.only debug testsuite here(includes basic testcase, - # i.e. refstack ipv6 vpn test cases from functest, HA test case - # from yardstick) + # 3.only debug testsuite here(refstack, ha, ipv6, bgpvpn) # 4.not used for release criteria or compliance, - # only to debug the dovetail tool bugs with fuel bgpvpn scenario - - trigger-builds: - - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-bgpvpn-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' + # only to debug the dovetail tool bugs with bgpvpn + # 5,only run against scenario os-odl_l2-bgpvpn-ha(regex used here, can extend to more scenarios future) + - conditional-step: + condition-kind: regex-match + regex: os-odl_l2-bgpvpn-ha + label: '{scenario}' + steps: + - trigger-builds: + - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' publishers: - email: - recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com matthew.lijun@huawei.com - + recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com - job-template: name: 'fuel-deploy-{pod}-daily-{stream}' diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index b180f591b..e7a487141 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -374,7 +374,10 @@ name: set-functest-env builders: - shell: - !include-raw: ./set-functest-env.sh + !include-raw: + - ./functest-env-presetup.sh + - ../../utils/fetch_os_creds.sh + - ./set-functest-env.sh - builder: name: functest-store-results diff --git a/jjb/functest/functest-env-presetup.sh b/jjb/functest/functest-env-presetup.sh new file mode 100755 index 000000000..7a9b09d2c --- /dev/null +++ b/jjb/functest/functest-env-presetup.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +# Fetch INSTALLER_IP for APEX deployments +if [[ ${INSTALLER_TYPE} == 'apex' ]]; then + echo "Gathering IP information for Apex installer VM" + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + if sudo virsh list | grep undercloud; then + echo "Installer VM detected" + undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + export INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) + export sshkey_vol="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" + sudo scp $ssh_options root@${INSTALLER_IP}:/home/stack/stackrc ${HOME}/stackrc + export stackrc_vol="-v ${HOME}/stackrc:/home/opnfv/functest/conf/stackrc" + + if sudo iptables -C FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then + sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable + fi + if sudo iptables -C FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then + sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable + fi + echo "Installer ip is ${INSTALLER_IP}" + else + echo "No available installer VM exists and no credentials provided...exiting" + exit 1 + fi +fi + diff --git a/jjb/functest/functest-project-jobs.yml b/jjb/functest/functest-project-jobs.yml index 14ad73a91..7036f20c0 100644 --- a/jjb/functest/functest-project-jobs.yml +++ b/jjb/functest/functest-project-jobs.yml @@ -88,4 +88,4 @@ name: functest-unit-tests-and-docs-build builders: - shell: | - $WORKSPACE/run_unit_tests.sh + cd $WORKSPACE && tox diff --git a/jjb/functest/functest-weekly-jobs.yml b/jjb/functest/functest-weekly-jobs.yml index f44f7b8aa..59d24cc87 100644 --- a/jjb/functest/functest-weekly-jobs.yml +++ b/jjb/functest/functest-weekly-jobs.yml @@ -115,7 +115,10 @@ - shell: !include-raw: ./functest-cleanup.sh - shell: - !include-raw: ./set-functest-env.sh + !include-raw: + - ./functest-env-presetup.sh + - ../../utils/fetch_os_creds.sh + - ./set-functest-env.sh - shell: !include-raw: ./functest-loop.sh - shell: diff --git a/jjb/functest/set-functest-env.sh b/jjb/functest/set-functest-env.sh index 89dabb056..558e2487d 100755 --- a/jjb/functest/set-functest-env.sh +++ b/jjb/functest/set-functest-env.sh @@ -1,50 +1,16 @@ #!/bin/bash set -e -[[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" -# LAB_CONFIG is used only for joid +set +u +set +o pipefail +[[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" +# Prepare OpenStack credentials volume if [[ ${INSTALLER_TYPE} == 'joid' ]]; then - # If production lab then creds may be retrieved dynamically - # creds are on the jumphost, always in the same folder rc_file_vol="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" - # If dev lab, credentials may not be the default ones, just provide a path to put them into docker - # replace the default one by the customized one provided by jenkins config -fi - -if [[ ${RC_FILE_PATH} != '' ]] && [[ -f ${RC_FILE_PATH} ]] ; then - echo "Credentials file detected: ${RC_FILE_PATH}" - # volume if credentials file path is given to Functest - rc_file_vol="-v ${RC_FILE_PATH}:/home/opnfv/functest/conf/openstack.creds" - RC_FLAG=1 -fi - - -if [[ ${INSTALLER_TYPE} == 'apex' ]]; then - ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - if sudo virsh list | grep undercloud; then - echo "Installer VM detected" - undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ - grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") - INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) - sshkey_vol="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" - sudo scp $ssh_options root@${INSTALLER_IP}:/home/stack/stackrc ${HOME}/stackrc - stackrc_vol="-v ${HOME}/stackrc:/home/opnfv/functest/conf/stackrc" - - if sudo iptables -C FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then - sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable - fi - if sudo iptables -C FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then - sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable - fi - elif [[ "$RC_FLAG" == 1 ]]; then - echo "No available installer VM, but credentials provided...continuing" - else - echo "No available installer VM exists and no credentials provided...exiting" - exit 1 - fi - +else + rc_file_vol="-v ${HOME}/opnfv-openrc.sh:/home/opnfv/functest/conf/openstack.creds" fi @@ -58,11 +24,13 @@ DEPLOY_TYPE=baremetal echo "Functest: Start Docker and prepare environment" -echo "Functest: Download images that will be used by test cases" -images_dir="${HOME}/opnfv/functest/images" -chmod +x ${WORKSPACE}/functest/ci/download_images.sh -${WORKSPACE}/functest/ci/download_images.sh ${images_dir} -images_vol="-v ${images_dir}:/home/opnfv/functest/images" +if [ "$BRANCH" != 'stable/danube' ]; then + echo "Functest: Download images that will be used by test cases" + images_dir="${HOME}/opnfv/functest/images" + chmod +x ${WORKSPACE}/functest/ci/download_images.sh + ${WORKSPACE}/functest/ci/download_images.sh ${images_dir} 2> ${redirect} + images_vol="-v ${images_dir}:/home/opnfv/functest/images" +fi dir_result="${HOME}/opnfv/functest/results/${BRANCH##*/}" mkdir -p ${dir_result} @@ -85,7 +53,11 @@ if [[ ${INSTALLER_TYPE} == 'compass' && ${DEPLOY_SCENARIO} == *'os-nosdn-openo-h envs=${env}" -e OPENO_MSB_ENDPOINT=${openo_msb_endpoint}" fi -volumes="${images_vol} ${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" +if [ "$BRANCH" != 'stable/danube' ]; then + volumes="${images_vol} ${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" +else + volumes="${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" +fi HOST_ARCH=$(uname -m) FUNCTEST_IMAGE="opnfv/functest" diff --git a/jjb/qtip/helpers/validate-deploy.sh b/jjb/qtip/helpers/validate-deploy.sh index 9f3dbe414..af8f8c200 100644 --- a/jjb/qtip/helpers/validate-deploy.sh +++ b/jjb/qtip/helpers/validate-deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash ############################################################################## -# Copyright (c) 2016 ZTE and others. +# Copyright (c) 2017 ZTE and others. # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -8,40 +8,20 @@ ############################################################################## set -e -envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} --e NODE_NAME=${NODE_NAME} -e CI_DEBUG=${CI_DEBUG}" -ramfs=/tmp/qtip/ramfs -cfg_dir=$(dirname $ramfs) -dir_imgstore="${HOME}/imgstore" -ramfs_volume="$ramfs:/mnt/ramfs" - echo "--------------------------------------------------------" echo "POD: $NODE_NAME" -echo "INSTALLER: $INSTALLER_TYPE" echo "Scenario: $DEPLOY_SCENARIO" +echo "INSTALLER: $INSTALLER_TYPE" +echo "INSTALLER_IP: $INSTALLER_IP" echo "--------------------------------------------------------" echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}" -docker pull opnfv/qtip:$DOCKER_TAG - -# use ramfs to fix docker socket connection issue with overlay mode in centos -if [ ! -d $ramfs ]; then - mkdir -p $ramfs -fi +docker pull opnfv/qtip:$DOCKER_TAG >/dev/null -if [ ! -z "$(df $ramfs | tail -n -1 | grep $ramfs)" ]; then - sudo mount -t tmpfs -o size=32M tmpfs $ramfs -fi - -# enable contro path in docker -cat <<EOF > ${cfg_dir}/ansible.cfg -[defaults] -callback_whitelist = profile_tasks -[ssh_connection] -control_path=/mnt/ramfs/ansible-ssh-%%h-%%p-%%r -EOF +envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} +-e POD_NAME=${NODE_NAME} -e SCENARIO=${DEPLOY_SCENARIO}" -cmd=" docker run -id -e $envs -v ${ramfs_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash" +cmd=" docker run -id -e $envs opnfv/qtip:${DOCKER_TAG} /bin/bash" echo "Qtip: Running docker command: ${cmd}" ${cmd} @@ -49,14 +29,12 @@ container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..." exit 1 -else - echo "The container ID is: ${container_id}" - QTIP_REPO=/home/opnfv/repos/qtip - docker cp ${cfg_dir}/ansible.cfg ${container_id}:/home/opnfv/.ansible.cfg -# TODO(zhihui_wu): use qtip cli to execute benchmark test in the future - docker exec -t ${container_id} bash -c "cd ${QTIP_REPO}/qtip/runner/ && - python runner.py -d /home/opnfv/qtip/results/ -b all" - fi +echo "The container ID is: ${container_id}" +QTIP_REPO=/home/opnfv/repos/qtip + +docker exec -t ${container_id} bash -c "bash ${QTIP_REPO}/tests/ci/run_ci.sh" + echo "Qtip done!" +exit 0
\ No newline at end of file diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index 166aea8f9..8c231c3e8 100644 --- a/jjb/releng/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -53,7 +53,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng|pharos|octopus|securedlab' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index 6a382a56c..458bbda3b 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -7,7 +7,9 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - +set -o errexit +set -o nounset +set -o pipefail usage() { echo "usage: $0 [-v] -d <destination> -i <installer_type> -a <installer_ip>" >&2 @@ -157,8 +159,8 @@ elif [ "$installer_type" == "compass" ]; then sshpass -p root scp 2>/dev/null $ssh_options root@${installer_ip}:~/admin-openrc.sh $dest_path &> /dev/null info "This file contains the mgmt keystone API, we need the public one for our rc file" - grep "OS_AUTH_URL.*v2" $dest_path > /dev/null 2>&1 - if [ $? -eq 0 ] ; then + + if grep "OS_AUTH_URL.*v2" $dest_path > /dev/null 2>&1 ; then public_ip=$(sshpass -p root ssh $ssh_options root@${installer_ip} \ "ssh ${controller_ip} 'source /opt/admin-openrc.sh; openstack endpoint show identity '" \ | grep publicurl | awk '{print $4}') @@ -223,5 +225,3 @@ fi echo "-------- Credentials: --------" cat $dest_path - -exit 0 diff --git a/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js b/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js index 3ce76c912..929f5433c 100644 --- a/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js +++ b/utils/test/testapi/3rd_party/static/testapi-ui/shared/alerts/alertModalFactory.js @@ -28,7 +28,7 @@ function raiseAlert($uibModal) { return function(mode, title, text) { $uibModal.open({ - templateUrl: '/shared/alerts/alertModal.html', + templateUrl: 'testapi-ui/shared/alerts/alertModal.html', controller: 'RaiseAlertModalController as alert', backdrop: true, keyboard: true, diff --git a/utils/test/testapi/opnfv_testapi/router/url_mappings.py b/utils/test/testapi/opnfv_testapi/router/url_mappings.py index aea85a4c4..a2312de09 100644 --- a/utils/test/testapi/opnfv_testapi/router/url_mappings.py +++ b/utils/test/testapi/opnfv_testapi/router/url_mappings.py @@ -55,7 +55,7 @@ mappings = [ (r"/api/v1/scenarios/([^/]+)", scenario_handlers.ScenarioGURHandler), # static path - (r'/(.*\.(css|png|gif|js|html|json|map))', + (r'/(.*\.(css|png|gif|js|html|json|map|woff2|woff|ttf))', tornado.web.StaticFileHandler, {'path': config.Config().static_path}), diff --git a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py index 73190841f..6a9d94eb2 100644 --- a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py +++ b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py @@ -32,6 +32,9 @@ class SigninHandler(base.BaseHandler): class SigninReturnHandler(base.BaseHandler): def get(self): + if self.get_query_argument(const.OPENID_MODE) == 'cancel': + self._auth_failure('Authentication canceled.') + openid = self.get_query_argument(const.OPENID_CLAIMED_ID) user_info = { 'openid': openid, @@ -44,6 +47,12 @@ class SigninReturnHandler(base.BaseHandler): self.set_secure_cookie('openid', openid) self.redirect(url=CONF.ui_url) + def _auth_failure(self, message): + params = {'message': message} + url = parse.urljoin(CONF.ui_url, + '/#/auth_failure?' + parse.urlencode(params)) + self.redirect(url) + class SignoutHandler(base.BaseHandler): def get(self): |