diff options
| -rw-r--r-- | jjb/apex/apex.yml | 181 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-build.sh | 36 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-deploy.sh | 49 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-download-artifact.sh | 29 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-lab-reconfig.sh | 72 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-upload-artifact.sh | 22 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-virtual-deploy.sh | 3 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-workspace-cleanup.sh | 7 | ||||
| -rw-r--r-- | jjb/fuel/fuel.yml | 410 | ||||
| -rw-r--r-- | jjb/functest/functest.yml | 315 | ||||
| -rw-r--r-- | jjb/genesis/genesis-foreman.yml | 2 | ||||
| -rw-r--r-- | jjb/genesis/genesis-fuel.yml | 26 | ||||
| -rw-r--r-- | jjb/opnfv/opnfv-docs.yml | 4 | ||||
| -rw-r--r-- | utils/test/result_collection_api/common/config.py | 6 | ||||
| -rw-r--r-- | utils/test/result_collection_api/config.ini | 8 | ||||
| -rw-r--r-- | utils/test/result_collection_api/resources/handlers.py | 120 | ||||
| -rw-r--r-- | utils/test/result_collection_api/resources/models.py | 32 | ||||
| -rw-r--r-- | utils/test/result_collection_api/result_collection_api.py | 7 |
18 files changed, 1046 insertions, 283 deletions
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 46e578b6e..65f042f50 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -1,8 +1,8 @@ - project: name: apex jobs: - - 'apex-verify' - - 'apex-merge' + - 'apex-verify-{stream}' + - 'apex-merge-{stream}' - 'apex-daily-{stream}' # stream: branch with - in place of / (eg. stable-arno) @@ -10,13 +10,16 @@ stream: - master: branch: 'master' + gs-pathname: '' + flags: '-fM' + disabled: false project: 'apex' - job-template: - name: 'apex-verify' + name: 'apex-verify-{stream}' - node: ericsson-build + node: opnfv-jump-1 project-type: freestyle @@ -27,14 +30,17 @@ artifactNumToKeep: -1 parameters: + - apex-parameter: + gs-pathname: '{gs-pathname}' + flags: '{flags}' - project-parameter: project: '{project}' - gerrit-parameter: - branch: 'master' + branch: '{branch}' - string: name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." scm: - gerrit-trigger-scm: @@ -64,19 +70,22 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' builders: - - shell: - echo "Hello World" + - 'apex-build' + - 'apex-workspace-cleanup' - job-template: - name: 'apex-merge' + name: 'apex-merge-{stream}' # builder-merge job to run JJB update # # This job's purpose is to update all the JJB - node: master + node: opnfv-jump-1 project-type: freestyle @@ -87,14 +96,17 @@ artifactNumToKeep: 5 parameters: + - apex-parameter: + gs-pathname: '{gs-pathname}' + flags: '{flags}' - project-parameter: project: '{project}' - gerrit-parameter: - branch: 'master' + branch: '{branch}' - string: name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." scm: - gerrit-trigger-scm: @@ -118,11 +130,13 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' builders: - - shell: - echo "Hello World" - + - 'apex-build' + - 'apex-workspace-cleanup' - job-template: name: 'apex-daily-{stream}' @@ -132,9 +146,9 @@ # Required Variables: # stream: branch with - in place of / (eg. stable) # branch: branch (eg. stable) - node: master + node: opnfv-jump-1 - disabled: true + disabled: false project-type: freestyle @@ -147,6 +161,15 @@ parameters: - project-parameter: project: '{project}' + - apex-parameter: + gs-pathname: '{gs-pathname}' + flags: '{flags}' + - gerrit-parameter: + branch: '{branch}' + - string: + name: GIT_BASE + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." scm: - git-scm: @@ -159,8 +182,122 @@ user: '{ssh-credentials}' triggers: - - timed: 'H H * * *' + - 'apex-{stream}' + + builders: + - 'apex-build' + - 'apex-upload-artifact' + - 'apex-workspace-cleanup' + +######################## +# parameter macros +######################## +- parameter: + name: apex-parameter + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/{gs-pathname} + description: "Directory where the cache to be used during the build is located." + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT/{gs-pathname} + description: "URL to Google Storage." + - string: + name: FLAGS + default: '{flags}' + description: "Build Flags" + +######################## +# builder macros +######################## +- builder: + name: 'apex-build' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + # log info to console + echo "Starting the build of Apex using OpenStack Master packages. This will take some time..." + echo "---------------------------------------------------------------------------------------" + echo "Build flags for this build are ${FLAGS}" + echo "---------------------------------------------------------------------------------------" + echo + # create the cache directory if it doesn't exist + [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY + # set OPNFV_ARTIFACT_VERSION + export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") + # start the build + cd $WORKSPACE/ci + ./build.sh $FLAGS -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY + # list the contents of BUILD_OUTPUT directory + ls -al $BUILD_DIRECTORY + # save information regarding artifact into file + ( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" + echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)" + echo "OPNFV_BUILD_URL=$BUILD_URL" + ) > $WORKSPACE/opnfv.properties + echo + echo "--------------------------------------------------------" + +- builder: + name: 'apex-workspace-cleanup' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # delete everything that is in $WORKSPACE + /bin/rm -rf $WORKSPACE +- builder: + name: 'apex-upload-artifact' builders: - - shell: - echo "Hello World" + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # log info to console + echo "Uploading the Apex artifact. This could take some time..." + echo "--------------------------------------------------------" + echo + + # source the opnfv.properties to get ARTIFACT_VERSION + source $WORKSPACE/opnfv.properties + + # upload artifact and additional files to google storage + gsutil cp $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 + gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 + gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 + + echo + echo "--------------------------------------------------------" + echo "Done!" + echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" + + +####################### +# trigger macros +######################## +- trigger: + name: 'apex-master' + triggers: + - timed: '0 3 * * *' diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh new file mode 100755 index 000000000..178a50c68 --- /dev/null +++ b/jjb/fuel/fuel-build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# log info to console +echo "Starting the build of $INSTALLER. This could take some time..." +echo "--------------------------------------------------------" +echo + +# create the cache directory if it doesn't exist +[[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY + +# set OPNFV_ARTIFACT_VERSION +export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") + +# start the build +cd $WORKSPACE/$INSTALLER/ci +./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY + +# list the build artifacts +ls -al $BUILD_DIRECTORY + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" + echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)" + echo "OPNFV_BUILD_URL=$BUILD_URL" +) > $WORKSPACE/opnfv.properties + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh new file mode 100755 index 000000000..bb0e2b15f --- /dev/null +++ b/jjb/fuel/fuel-deploy.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# source the file so we get OPNFV vars +source latest.properties + +# echo the info about artifact that is used during the deployment +echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3) for deployment" + +# create TMPDIR if it doesn't exist +export TMPDIR=$HOME/tmpdir +[[ -d $TMPDIR ]] || mkdir -p $TMPDIR + +# change permissions down to TMPDIR +chmod a+x $HOME +chmod a+x $TMPDIR + +# set CONFDIR, BRIDGE +export CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2 +export BRIDGE=pxebr + +# clone genesis repo and checkout the SR1 tag +echo "Cloning genesis repo" +cd $WORKSPACE +GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/genesis genesis +cd genesis +git checkout arno.2015.2.0 + +# cleanup first +sudo $WORKSPACE/genesis/common/ci/clean.sh -base_config $WORKSPACE/genesis/foreman/ci/inventory/lf_pod2_ksgen_settings.yml + +# prepare for Fuel Deployment +sudo $WORKSPACE/genesis/common/ci/setup.sh + +# log info to console +echo "Starting the deployment using $INSTALLER. This could take some time..." +echo "--------------------------------------------------------" +echo + +# start the deployment +echo "Issuing command" +echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh" +sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh new file mode 100755 index 000000000..6eb1ba463 --- /dev/null +++ b/jjb/fuel/fuel-download-artifact.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# get the latest.properties file in order to get info regarding latest artifact +curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties + +# check if we got the file +[[ -f latest.properties ]] || exit 1 + +# source the file so we get OPNFV vars +source latest.properties + +# log info to console +echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL" +echo "This could take some time..." +echo "--------------------------------------------------------" +echo + +# download the file +curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 + +# list the file +ls -al $WORKSPACE/opnfv.iso + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/fuel/fuel-lab-reconfig.sh b/jjb/fuel/fuel-lab-reconfig.sh new file mode 100755 index 000000000..a79bc2c51 --- /dev/null +++ b/jjb/fuel/fuel-lab-reconfig.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# check to see if ucs login info file exists +if [ -e ~/.ssh/ucs_creds ];then + source ~/.ssh/ucs_creds +else + echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting" + exit 1 +fi + +# clone releng +echo "Cloning releng repo..." +if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then + echo "Unable to clone releng repo...Exiting" + exit 1 +fi + +# log info to console +echo "Starting the lab reconfiguration for $INSTALLER..." +echo "--------------------------------------------------------" +echo + +# create venv +$WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh + +# disable nounset because 'activate' script contains unbound variable(s) +set +o nounset +# enter venv +source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate +# set nounset back again +set -o nounset + +# verify we are in venv +if [[ ! $(which python | grep venv) ]]; then + echo "Unable to activate venv...Exiting" + exit 1 +fi + +python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/fuel.yaml + +# while undergoing reboot +sleep 30 + +# check to see if slave is back up +ping_counter=0 +ping_flag=0 +while [ "$ping_counter" -lt 20 ]; do + if [[ $(ping -c 5 172.30.10.72) ]]; then + ping_flag=1 + break + fi + ((ping_counter++)) + sleep 10 +done + +if [ "$ping_flag" -eq 1 ]; then + echo "Slave is pingable, now wait 180 seconds for services to start" + sleep 180 +else + echo "Slave did not come back up after reboot: please check opnfv-jump-2" + exit 1 +fi + +set +o nounset +deactivate + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/fuel/fuel-upload-artifact.sh b/jjb/fuel/fuel-upload-artifact.sh new file mode 100755 index 000000000..3b700c649 --- /dev/null +++ b/jjb/fuel/fuel-upload-artifact.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# log info to console +echo "Uploading the $INSTALLER artifact. This could take some time..." +echo "--------------------------------------------------------" +echo + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" diff --git a/jjb/fuel/fuel-virtual-deploy.sh b/jjb/fuel/fuel-virtual-deploy.sh new file mode 100755 index 000000000..4e0b2dc47 --- /dev/null +++ b/jjb/fuel/fuel-virtual-deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Placeholder for Fuel virtual deploy" diff --git a/jjb/fuel/fuel-workspace-cleanup.sh b/jjb/fuel/fuel-workspace-cleanup.sh new file mode 100755 index 000000000..4e1a3a727 --- /dev/null +++ b/jjb/fuel/fuel-workspace-cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# delete the $WORKSPACE to open some space +/bin/rm -rf $WORKSPACE diff --git a/jjb/fuel/fuel.yml b/jjb/fuel/fuel.yml index e108051ea..503aa1b82 100644 --- a/jjb/fuel/fuel.yml +++ b/jjb/fuel/fuel.yml @@ -1,24 +1,46 @@ +######################## +# Job configuration for fuel +######################## - project: + name: fuel + + project: 'fuel' + + installer: 'fuel' + jobs: - - 'fuel-verify' - - 'fuel-merge' + - 'fuel-verify-build-{stream}' + - 'fuel-verify-virtual-deploy-{stream}' + - 'fuel-merge-build-{stream}' + - 'fuel-merge-virtual-deploy-{stream}' - 'fuel-daily-{stream}' + - 'fuel-build-{stream}' + - 'fuel-deploy-{stream}' + - 'fuel-lab-reconfig-{stream}' - # stream: branch with - in place of / (eg. stable-arno) - # branch: branch (eg. stable/arno) stream: - master: branch: 'master' + gs-pathname: '' - project: 'fuel' +######################## +# job templates +######################## - job-template: - name: 'fuel-verify' + name: 'fuel-verify-build-{stream}' + + project-type: freestyle node: ericsson-build - project-type: freestyle + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 3 logrotate: daysToKeep: 30 @@ -30,11 +52,10 @@ - project-parameter: project: '{project}' - gerrit-parameter: - branch: 'master' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + branch: '{branch}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' scm: - gerrit-trigger-scm: @@ -60,26 +81,101 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'ANT' - project-pattern: 'fuel' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' - branch-pattern: '**/master' + branch-pattern: '**/{branch}' builders: - shell: - echo "Hello World" + !include-raw ./fuel-build.sh +# - shell: +# !include-raw ./fuel-upload-artifact.sh +# - shell: +# !include-raw ./fuel-workspace-cleanup.sh - job-template: - name: 'fuel-merge' + name: 'fuel-verify-virtual-deploy-{stream}' - # builder-merge job to run JJB update - # - # This job's purpose is to update all the JJB + project-type: freestyle - node: master + node: ericsson-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 2 + max-per-node: 1 + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + user: '{ssh-credentials}' + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + dependency-jobs: 'fuel-verify-build-{stream}' + + builders: +# - shell: +# !include-raw ./fuel-download-artifact.sh + - shell: + !include-raw ./fuel-virtual-deploy.sh +# - shell: +# !include-raw ./fuel-workspace-cleanup.sh + +- job-template: + name: 'fuel-merge-build-{stream}' project-type: freestyle + node: ericsson-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 2 + logrotate: daysToKeep: 30 numToKeep: 40 @@ -90,13 +186,70 @@ - project-parameter: project: '{project}' - gerrit-parameter: - branch: 'master' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + branch: '{branch}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '' + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + choosing-strategy: 'default' + + wrappers: + - ssh-agent-credentials: + user: '{ssh-credentials}' + + triggers: + - gerrit: + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + + builders: + - shell: + !include-raw ./fuel-build.sh +# - shell: +# !include-raw ./fuel-upload-artifact.sh +# - shell: +# !include-raw ./fuel-workspace-cleanup.sh + +- job-template: + name: 'fuel-merge-virtual-deploy-{stream}' + + project-type: freestyle + + node: ericsson-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 2 + max-per-node: 1 + logrotate: + daysToKeep: 30 + numToKeep: 40 + artifactDaysToKeep: -1 + artifactNumToKeep: 5 + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '' scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' @@ -115,29 +268,44 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'ANT' - project-pattern: 'fuel' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' - branch-pattern: '**/master' + branch-pattern: '**/{branch}' + dependency-jobs: 'fuel-merge-build-{stream}' builders: +# - shell: +# !include-raw ./fuel-download-artifact.sh - shell: - echo "Hello World" - + !include-raw ./fuel-virtual-deploy.sh +# - shell: +# !include-raw ./fuel-workspace-cleanup.sh - job-template: name: 'fuel-daily-{stream}' - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: master + project-type: freestyle - disabled: true + node: ericsson-build - project-type: freestyle + disabled: false + + triggers: + - 'fuel-{strem}-trigger' + + parameters: + - project-parameter: + project: '{project}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' logrotate: daysToKeep: '{build-days-to-keep}' @@ -145,9 +313,45 @@ artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' + builders: + - trigger-builds: + - project: 'fuel-build-{stream}' + git-revision: true + block: true + - trigger-builds: + - project: 'fuel-deploy-{stream}' + git-revision: true + block: true + - trigger-builds: + - project: 'functest-fuel-{stream}' + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + + triggers: + - 'fuel-{stream}-daily-trigger' + +- job-template: + name: 'fuel-build-{stream}' + + project-type: freestyle + + node: ericsson-build + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + parameters: - project-parameter: project: '{project}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' scm: - git-scm: @@ -155,13 +359,133 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' + builders: + - shell: + !include-raw ./fuel-build.sh + - shell: + !include-raw ./fuel-upload-artifact.sh + - shell: + !include-raw ./fuel-workspace-cleanup.sh - triggers: - - timed: 'H H * * *' + publishers: + - email: + recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com + +- job-template: + name: 'fuel-deploy-{stream}' + + project-type: freestyle + + disabled: false + + node: opnfv-jump-2 + + parameters: + - project-parameter: + project: '{project}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - string: + name: GIT_BASE + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "POD2 has some issues with cloning using https so that's why GIT_BASE is overriden here again." + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + builders: + - shell: + !include-raw ./fuel-download-artifact.sh + - shell: + !include-raw ./fuel-deploy.sh + + publishers: + - email: + recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com + +- job-template: + name: 'fuel-lab-reconfig-{stream}' + + project-type: freestyle + + disabled: true + + node: master + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + parameters: + - project-parameter: + project: '{project}' + - fuel-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - string: + name: GIT_BASE + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + properties: + - build-blocker: + use-build-blocker: true + blocking-jobs: + - "apex-daily.*" builders: - shell: - echo "Hello World" + !include-raw ./fuel-lab-reconfig.sh + +######################## +# parameter macros +######################## +- parameter: + name: fuel-parameter + parameters: + - string: + name: INSTALLER + default: '{installer}' + description: "Installer to use." + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER + description: "Directory where the cache to be used during the build is located." + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." + +######################## +# trigger macros +######################## +- trigger: + name: 'fuel-master-daily-trigger' + triggers: + - timed: '0 3 * * *' diff --git a/jjb/functest/functest.yml b/jjb/functest/functest.yml index 2d57102ad..6268f27d2 100644 --- a/jjb/functest/functest.yml +++ b/jjb/functest/functest.yml @@ -9,15 +9,18 @@ node: 'opnfv-jump-2' installer_type: 'fuel' installer_ip: '10.20.0.2' + functest_repo_dir: '/home/opnfv/repos/functest' - foreman: node: 'opnfv-jump-2' installer_type: 'foreman' installer_ip: '172.30.10.73' + functest_repo_dir: '/home/opnfv/repos/functest' pod: - orange-build: node: 'orange-build' installer_type: 'foreman' installer_ip: '172.30.10.73' + functest_repo_dir: '/home/opnfv/repos/functest' jobs: - 'set-functest-env-{pod}' @@ -29,7 +32,7 @@ - 'functest-openstack-bench-test-{pod}' - 'functest-openstack-tempest-smoke-test-{pod}' - 'functest-odl-test-{pod}' - - 'functest-vims-test-{pod}' + - 'functest-vims-{installer}-{stream}' - 'functest-vping-test-{pod}' - 'functest-daily-{stream}' - 'functest-merge-{stream}' @@ -64,26 +67,9 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e + - 'set-functest-env' - echo "Functest: prepare Functest environment" - mkdir -p $HOME/functest/ - # source openstack vars - if [[ ! -f $HOME/opnfv-openrc.sh ]]; then - echo "Unable to access file $HOME/opnfv-openrc.sh" - exit 1 - fi - source $HOME/opnfv-openrc.sh - # WORKSPACE is the root of the functest repo - # go into WORKSPACE where the functest repo is cloned - cd $WORKSPACE - - # prepare - echo "Functest: prepare Functest environment" - python $WORKSPACE/testcases/config_functest.py --debug $WORKSPACE/ start - job-template: name: clean-functest-env-{pod} @@ -99,24 +85,8 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e - - # collect results - echo "Functest: copy results and clean Functest environment" - mkdir -p $HOME/functest/results - - # save ODL results - cp -Rf $WORSPACE/testcases/Controllers/ODL/CI/logs $HOME/functest/results/ODL - - # save tempest.conf for further troubleshooting - cp $HOME/.rally/tempest/for-deployment-*/tempest.conf $HOME/functest/results + - 'functest-cleanup' - # cleanup - # source openstack vars - source $HOME/opnfv-openrc.sh - python $WORKSPACE/testcases/config_functest.py --debug --force $WORKSPACE/ clean - job-template: name: set-functest-env-{installer}-{stream} @@ -132,26 +102,8 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e - - echo "Functest: prepare Functest environment" - mkdir -p $HOME/functest/ - # source openstack vars - if [[ ! -f $HOME/opnfv-openrc.sh ]]; then - echo "Unable to access file $HOME/opnfv-openrc.sh" - exit 1 - fi - source $HOME/opnfv-openrc.sh + - 'set-functest-env' - # WORKSPACE is the root of the functest repo - # go into WORKSPACE where the functest repo is cloned - cd $WORKSPACE - - # prepare - echo "Functest: prepare Functest environment" - python $WORKSPACE/testcases/config_functest.py --debug $WORKSPACE/ start - job-template: name: clean-functest-env-{installer}-{stream} @@ -167,24 +119,7 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e - - # collect results - echo "Functest: copy results and clean Functest environment" - mkdir -p $HOME/functest/results - - # save ODL results - cp -Rf $WORSPACE/testcases/Controllers/ODL/CI/logs $HOME/functest/results/ODL - - # save tempest.conf for further troubleshooting - cp $HOME/.rally/tempest/for-deployment-*/tempest.conf $HOME/functest/results - - # cleanup - # source openstack vars - source $HOME/opnfv-openrc.sh - python $WORKSPACE/testcases/config_functest.py --debug --force $WORKSPACE/ clean + - 'functest-cleanup' - job-template: name: functest-openstack-bench-test-{pod} @@ -200,12 +135,8 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e + - 'functest-rally-bench' - echo "Functest: run Functest Rally Bench suites" - python $WORKSPACE/testcases/VIM/OpenStack/CI/libraries/run_rally.py --debug $WORKSPACE/ all - job-template: name: functest-openstack-tempest-smoke-test-{pod} @@ -221,14 +152,7 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e - - # tempest - echo "Functest: run Tempest suite" - rally verify start smoke - rally verify list + - 'functest-tempest-smoke' - job-template: name: functest-odl-test-{pod} @@ -245,14 +169,11 @@ builders: - shell: | - #!/bin/bash - set +e + - 'functest-odl' - echo "Functest: run ODL suite" - $WORKSPACE/testcases/Controllers/ODL/CI/start_tests.sh - job-template: - name: functest-vims-test-{pod} + name: functest-vping-test-{pod} project-type: freestyle @@ -265,17 +186,39 @@ artifactNumToKeep: -1 builders: - - shell: | - echo "put your CI scenario here" + - 'functest-vping' - job-template: - name: functest-vping-test-{pod} + name: functest-{pod} project-type: freestyle node: '{pod}' + parameters: + - project-parameter: + project: '{project}' + - string: + name: INSTALLER_TYPE + default: '{installer_type}' + description: "Installer name that is used for deployment." + - string: + name: INSTALLER_IP + default: '{installer_ip}' + description: "Installer IP." + - string: + name: FUNCTEST_REPO_DIR + default: '{functest_repo_dir}' + description: "Directory where the root functest repo is cloned in the docker." + - '{pod}' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: master + logrotate: daysToKeep: 30 numToKeep: 10 @@ -283,20 +226,16 @@ artifactNumToKeep: -1 builders: - - shell: | - #!/bin/bash - set +e - - # vPing - echo "Functest: run vPing" - python $WORKSPACE/testcases/vPing/CI/libraries/vPing.py --debug $WORKSPACE/ -r + - 'functest-all' + - 'functest-store-results' + - 'functest-cleanup' - job-template: - name: functest-{pod} + name: functest-{installer}-{stream} project-type: freestyle - node: '{pod}' + node: '{node}' parameters: - project-parameter: @@ -309,7 +248,14 @@ name: INSTALLER_IP default: '{installer_ip}' description: "Installer IP." - - '{pod}' + - string: + name: FUNCTEST_REPO_DIR + default: '{functest_repo_dir}' + description: "Directory where the root functest repo is cloned in the docker." + - string: + name: GIT_BASE + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Override GIT_BASE" scm: - git-scm: @@ -324,13 +270,12 @@ artifactNumToKeep: -1 builders: - - 'functest-fetch-os-creds' - 'functest-all' - 'functest-store-results' - 'functest-cleanup' - job-template: - name: functest-{installer}-{stream} + name: functest-vims-{installer}-{stream} project-type: freestyle @@ -348,6 +293,10 @@ default: '{installer_ip}' description: "Installer IP." - string: + name: FUNCTEST_REPO_DIR + default: '{functest_repo_dir}' + description: "Directory where the root functest repo is cloned in the docker." + - string: name: GIT_BASE default: ssh://gerrit.opnfv.org:29418/$PROJECT description: "Override GIT_BASE" @@ -365,10 +314,7 @@ artifactNumToKeep: -1 builders: - - 'functest-fetch-os-creds' - - 'functest-all' - - 'functest-store-results' - - 'functest-cleanup' + - 'functest-vims' - job-template: name: 'functest-daily-{stream}' @@ -578,70 +524,99 @@ #!/bin/bash set +e - echo "Functest: prepare Functest environment" - mkdir -p $HOME/functest/ - - # source openstack vars - if [ ! -f $HOME/opnfv-openrc.sh ]; then - echo "Credentials file not found in "$HOME/opnfv-openrc.sh - exit 1 - fi - echo "Sourcing the OpenStack credentials in"$HOME/opnfv-openrc.sh - source $HOME/opnfv-openrc.sh - - # WORKSPACE is the root of the functest repo - # go into WORKSPACE where the functest repo is cloned - cd $WORKSPACE - - # prepare - echo "Functest: prepare Functest environment" - python $WORKSPACE/testcases/config_functest.py --debug $WORKSPACE/ start - if [ $? != 0 ]; then - echo "Error when configuring Functest environment" - exit 1 - fi - - # vPing + # Remove any docker containers leftovers + $ docker ps | grep opnfv/functest | awk '{print $1}' | xargs docker stop &>/dev/null + $ docker ps -a | grep opnfv/functest | awk '{print $1}' | xargs docker rm &>/dev/null + $ docker images | grep opnfv/functest | awk '{print $3}' | xargs docker rmi &>/dev/null + + # Pull and store fresh image from Dockerhub + $ docker pull opnfv/functest + + # Run Docker and Functest tests + cmd="${FUNCTEST_REPO_DIR}/docker/start.sh" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd + + +- builder: + name: set-functest-env + builders: + - shell: | + #!/bin/bash + set +e + + echo "Functest: spawn Docker and run Functest suite" + cmd="${FUNCTEST_REPO_DIR}/docker/start.sh" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd + + +- builder: + name: functest-vping + builders: + - shell: | + #!/bin/bash + set +e + echo "Functest: run vPing" - python $WORKSPACE/testcases/vPing/CI/libraries/vPing.py --debug $WORKSPACE/ -r + cmd="python ${FUNCTEST_REPO_DIR}/testcases/vPing/CI/libraries/vPing.py \ + --debug ${FUNCTEST_REPO_DIR}/ -r" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd + +- builder: + name: functest-odl + builders: + - shell: | + #!/bin/bash + set +e - # ODL echo "Functest: run ODL suite" + cmd="${FUNCTEST_REPO_DIR}/testcases/Controllers/ODL/CI/start_tests.sh" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd - if [ $INSTALLER_TYPE == "fuel" ]; then - odl_ip=$(keystone catalog --service network | grep publicURL | cut -f3 -d"/" | cut -f1 -d":") - neutron_ip=$(keystone catalog --service identity | grep publicURL | cut -f3 -d"/" | cut -f1 -d":") - usr_name=$(env | grep OS | grep OS_USERNAME | cut -f2 -d'=') - pass=$(env | grep OS | grep OS_PASSWORD | cut -f2 -d'=') - odl_port=8181 - ODL_PORT=$odl_port ODL_IP=$odl_ip NEUTRON_IP=$neutron_ip USR_NAME=$usr_name PASS=$pass \ - $WORKSPACE/testcases/Controllers/ODL/CI/start_tests.sh - elif [ $INSTALLER_TYPE == "foreman" ]; then - #odl_port=8081 - $WORKSPACE/testcases/Controllers/ODL/CI/start_tests.sh - else - echo "INSTALLER_TYPE not valid." - exit 1 - fi - - # rally - echo "Functest: run Functest Rally Bench suites" - python $WORKSPACE/testcases/VIM/OpenStack/CI/libraries/run_rally.py --debug $WORKSPACE/ all +- builder: + name: functest-tempest-smoke + builders: + - shell: | + #!/bin/bash + set +e # tempest echo "Functest: run Tempest suite" - rally verify start smoke - rally verify list + cmd="rally verify start smoke; rally verify list;" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd - # collect results - echo "Functest: copy results and clean Functest environment" - mkdir -p $HOME/functest/results - # save ODL results - cp -Rf $WORKSPACE/testcases/Controllers/ODL/CI/logs $HOME/functest/results/ODL - # save tempest.conf for further troubleshooting - cp $HOME/.rally/tempest/for-deployment-*/tempest.conf $HOME/functest/results +- builder: + name: functest-rally-bench + builders: + - shell: | + #!/bin/bash + set +e + + echo "Functest: run Functest Rally Bench suites" + cmd="python ${FUNCTEST_REPO_DIR}/testcases/VIM/OpenStack/CI/libraries/run_rally.py \ + --debug ${FUNCTEST_REPO_DIR}/ all" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd + + +- builder: + name: functest-vims + builders: + - shell: | + #!/bin/bash + set +e + cmd="python ${FUNCTEST_REPO_DIR}/testcases/vIMS/CI/vIMS.py \ + --debug ${FUNCTEST_REPO_DIR}/" + docker run -t -e "INSTALLER_TYPE=${INSTALLER_TYPE}" \ + -e "INSTALLER_IP=${INSTALLER_IP}" opnfv/functest $cmd + + - builder: name: functest-store-results @@ -656,12 +631,8 @@ #!/bin/bash set +e - # cleanup - source $HOME/opnfv-openrc.sh - python $WORKSPACE/testcases/config_functest.py --debug --force $WORKSPACE/ clean + # cleanup: remove any docker containers leftovers + $ docker ps | grep opnfv/functest | awk '{print $1}' | xargs docker stop &>/dev/null + $ docker ps -a | grep opnfv/functest | awk '{print $1}' | xargs docker rm &>/dev/null + $ docker images | grep opnfv/functest | awk '{print $3}' | xargs docker rmi &>/dev/null -- builder: - name: functest-fetch-os-creds - builders: - - shell: - !include-raw ../../utils/fetch_os_creds.sh diff --git a/jjb/genesis/genesis-foreman.yml b/jjb/genesis/genesis-foreman.yml index 77cc76c9d..7ee3e8ff0 100644 --- a/jjb/genesis/genesis-foreman.yml +++ b/jjb/genesis/genesis-foreman.yml @@ -29,7 +29,7 @@ - stable-arno: branch: 'stable/arno' gs-pathname: '/arno' - disabled: false + disabled: true ######################## # job templates ######################## diff --git a/jjb/genesis/genesis-fuel.yml b/jjb/genesis/genesis-fuel.yml index 66027768c..a957a693b 100644 --- a/jjb/genesis/genesis-fuel.yml +++ b/jjb/genesis/genesis-fuel.yml @@ -29,7 +29,7 @@ - stable-arno: branch: 'stable/arno' gs-pathname: '/arno' - disabled: false + disabled: true ######################## # job templates @@ -60,7 +60,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' @@ -146,7 +146,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '' @@ -211,12 +211,12 @@ disabled: '{obj:disabled}' triggers: - - 'fuel-{stream}' + - 'genesis-fuel-{stream}-trigger' parameters: - project-parameter: project: '{project}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' @@ -283,7 +283,7 @@ parameters: - project-parameter: project: '{project}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' - string: @@ -328,7 +328,7 @@ parameters: - project-parameter: project: '{project}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' @@ -359,7 +359,7 @@ parameters: - project-parameter: project: '{project}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' - string: @@ -405,7 +405,7 @@ parameters: - project-parameter: project: '{project}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' - string: @@ -440,7 +440,7 @@ parameters: - project-parameter: project: '{project}' - - fuel-parameter: + - genesis-fuel-parameter: installer: '{installer}' gs-pathname: '' @@ -463,7 +463,7 @@ # parameter macros ######################## - parameter: - name: fuel-parameter + name: genesis-fuel-parameter parameters: - string: name: INSTALLER @@ -751,11 +751,11 @@ # trigger macros ######################## - trigger: - name: 'fuel-master' + name: 'genesis-fuel-master-trigger' triggers: - timed: '0 5 * * *' - trigger: - name: 'fuel-stable-arno' + name: 'genesis-fuel-stable-arno-trigger' triggers: - timed: '0 7 * * *' diff --git a/jjb/opnfv/opnfv-docs.yml b/jjb/opnfv/opnfv-docs.yml index 4ed6bdeb1..42c3d1626 100644 --- a/jjb/opnfv/opnfv-docs.yml +++ b/jjb/opnfv/opnfv-docs.yml @@ -65,7 +65,7 @@ projects: - project-compare-type: 'REG_EXP' #Left out arno projects oscar|octopus|functest|genesis|pharos|opnfvdocs - project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sfc|vnf_forwarding_graph' + project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sfc|vnf_forwarding_graph|enfv|pinpoint' branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' @@ -114,7 +114,7 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'releng' + project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sfc|vnf_forwarding_graph|enfv|pinpoint' branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' diff --git a/utils/test/result_collection_api/common/config.py b/utils/test/result_collection_api/common/config.py index a0d0757a5..a62e80593 100644 --- a/utils/test/result_collection_api/common/config.py +++ b/utils/test/result_collection_api/common/config.py @@ -47,6 +47,7 @@ class APIConfig: def __init__(self): self._default_config_location = "config.ini" self.mongo_url = None + self.mongo_dbname = None self.api_port = None self.api_debug_on = None self._parser = None @@ -87,13 +88,18 @@ class APIConfig: # Linking attributes to keys from file with their sections obj.mongo_url = obj._get_parameter("mongo", "url") + obj.mongo_dbname = obj._get_parameter("mongo", "dbname") + obj.api_port = obj._get_int_parameter("api", "port") obj.api_debug_on = obj._get_bool_parameter("api", "debug") + return obj def __str__(self): return "mongo_url = %s \n" \ + "mongo_dbname = %s \n" \ "api_port = %s \n" \ "api_debug_on = %s \n" % (self.mongo_url, + self.mongo_dbname, self.api_port, self.api_debug_on) diff --git a/utils/test/result_collection_api/config.ini b/utils/test/result_collection_api/config.ini index e00b56c5f..f703cc6c4 100644 --- a/utils/test/result_collection_api/config.ini +++ b/utils/test/result_collection_api/config.ini @@ -1,10 +1,12 @@ +# to add a new parameter in the config file, +# the CONF object in config.ini must be updated [mongo] # URL of the mongo DB # Mongo auth url => mongodb://user1:pwd1@host1/?authSource=db1 -url = mongodb://127.0.0.1:27017/test_results_collection - +url = mongodb://127.0.0.1:27017/ +dbname = test_results_collection [api] # Listening port -port = 80 +port = 8000 # With debug_on set to true, error traces will be shown in HTTP responses debug = True
\ No newline at end of file diff --git a/utils/test/result_collection_api/resources/handlers.py b/utils/test/result_collection_api/resources/handlers.py index 3faba5aeb..f8b26d090 100644 --- a/utils/test/result_collection_api/resources/handlers.py +++ b/utils/test/result_collection_api/resources/handlers.py @@ -11,7 +11,7 @@ import json from tornado.web import RequestHandler, asynchronous, HTTPError from tornado import gen -from datetime import datetime +from datetime import datetime, timedelta from models import Pod, TestProject, TestCase, TestResult from common.constants import DEFAULT_REPRESENTATION, HTTP_BAD_REQUEST, \ @@ -60,6 +60,8 @@ class PodHandler(GenericApiHandler): """ Handle the requests about the POD Platforms HTTP Methdods : - GET : Get PODS + - POST : Create a pod + - DELETE : DELETE POD """ def initialize(self): @@ -68,19 +70,15 @@ class PodHandler(GenericApiHandler): @asynchronous @gen.coroutine - def get(self, pod_id=None): + def get(self, pod_name=None): """ Get all pods or a single pod :param pod_id: """ - - if pod_id is None: - pod_id = "" - get_request = dict() - if len(pod_id) > 0: - get_request["_id"] = int(pod_id) + if pod_name is not None: + get_request["name"] = pod_name res = [] cursor = self.db.pod.find(get_request) @@ -98,6 +96,69 @@ class PodHandler(GenericApiHandler): self.finish_request(answer) + @asynchronous + @gen.coroutine + def post(self): + """ Create a POD""" + + if self.json_args is None: + raise HTTPError(HTTP_BAD_REQUEST) + + query = {"name": self.json_args.get("name")} + + # check for existing name in db + mongo_dict = yield self.db.pod.find_one(query) + if mongo_dict is not None: + raise HTTPError(HTTP_FORBIDDEN, + "{} already exists as a pod".format( + self.json_args.get("name"))) + + pod = Pod.pod_from_dict(self.json_args) + pod.creation_date = datetime.now() + + future = self.db.pod.insert(pod.format()) + result = yield future + pod._id = result + + meta = dict() + meta["success"] = True + meta["uri"] = "/pods/{}".format(pod.name) + + answer = dict() + answer["pod"] = pod.format_http() + answer["meta"] = meta + + self.finish_request(answer) + + @asynchronous + @gen.coroutine + def delete(self, pod_name): + """ Remove a POD + + # check for an existing pod to be deleted + mongo_dict = yield self.db.pod.find_one( + {'name': pod_name}) + pod = TestProject.pod(mongo_dict) + if pod is None: + raise HTTPError(HTTP_NOT_FOUND, + "{} could not be found as a pod to be deleted" + .format(pod_name)) + + # just delete it, or maybe save it elsewhere in a future + res = yield self.db.test_projects.remove( + {'name': pod_name}) + + meta = dict() + meta["success"] = True + meta["deletion-data"] = res + + answer = dict() + answer["meta"] = meta + + self.finish_request(answer) + """ + pass + class TestProjectHandler(GenericApiHandler): """ @@ -440,18 +501,26 @@ class TestResultsHandler(GenericApiHandler): Available filters for this request are : - project : project name - case : case name - - pod : pod ID + - pod : pod name + - version : platform version (Arno-R1, ...) + - installer (fuel, ...) + - period : x (x last days) + :param result_id: Get a result by ID :raise HTTPError - GET /results/project=functest&case=keystone.catalog&pod=1 + GET /results/project=functest&case=vPing&version=Arno-R1 \ + &pod=pod_name&period=15 => get results with optional filters """ project_arg = self.get_query_argument("project", None) case_arg = self.get_query_argument("case", None) pod_arg = self.get_query_argument("pod", None) + version_arg = self.get_query_argument("version", None) + installer_arg = self.get_query_argument("installer", None) + period_arg = self.get_query_argument("period", None) # prepare request get_request = dict() @@ -463,15 +532,34 @@ class TestResultsHandler(GenericApiHandler): get_request["case_name"] = case_arg if pod_arg is not None: - get_request["pod_id"] = int(pod_arg) + get_request["pod_name"] = pod_arg + + if version_arg is not None: + get_request["version"] = version_arg + + if installer_arg is not None: + get_request["installer"] = installer_arg + + if period_arg is not None: + try: + period_arg = int(period_arg) + except: + raise HTTPError(HTTP_BAD_REQUEST) + + if period_arg > 0: + period = datetime.now() - timedelta(days=period_arg) + obj = {"$gte": period} + get_request["creation_date"] = obj else: get_request["_id"] = result_id + print get_request res = [] # fetching results cursor = self.db.test_results.find(get_request) while (yield cursor.fetch_next): - test_result = TestResult.test_result_from_dict(cursor.next_object()) + test_result = TestResult.test_result_from_dict( + cursor.next_object()) res.append(test_result.format_http()) # building meta object @@ -502,7 +590,9 @@ class TestResultsHandler(GenericApiHandler): raise HTTPError(HTTP_BAD_REQUEST) if self.json_args.get("case_name") is None: raise HTTPError(HTTP_BAD_REQUEST) - if self.json_args.get("pod_id") is None: + # check for pod_name instead of id, + # keeping id for current implementations + if self.json_args.get("pod_name") is None: raise HTTPError(HTTP_BAD_REQUEST) # TODO : replace checks with jsonschema @@ -524,11 +614,11 @@ class TestResultsHandler(GenericApiHandler): # check for pod mongo_dict = yield self.db.pod.find_one( - {"_id": self.json_args.get("pod_id")}) + {"name": self.json_args.get("pod_name")}) if mongo_dict is None: raise HTTPError(HTTP_NOT_FOUND, "Could not find POD [{}] " - .format(self.json_args.get("pod_id"))) + .format(self.json_args.get("pod_name"))) # convert payload to object test_result = TestResult.test_result_from_dict(self.json_args) diff --git a/utils/test/result_collection_api/resources/models.py b/utils/test/result_collection_api/resources/models.py index 82c17fda3..3b4d843f3 100644 --- a/utils/test/result_collection_api/resources/models.py +++ b/utils/test/result_collection_api/resources/models.py @@ -22,13 +22,19 @@ class Pod: p = Pod() p._id = pod_dict.get('_id') - p.creation_date = pod_dict.get('creation_date') + p.creation_date = str(pod_dict.get('creation_date')) p.name = pod_dict.get('name') return p def format(self): return { - "_id": self._id, + "name": self.name, + "creation_date": str(self.creation_date), + } + + def format_http(self): + return { + "_id": str(self._id), "name": self.name, "creation_date": str(self.creation_date), } @@ -82,6 +88,7 @@ class TestCase: self.name = None self.project_name = None self.description = None + self.url = None self.creation_date = None @staticmethod @@ -96,6 +103,7 @@ class TestCase: t.creation_date = testcase_dict.get('creation_date') t.name = testcase_dict.get('name') t.description = testcase_dict.get('description') + t.url = testcase_dict.get('url') return t @@ -104,7 +112,8 @@ class TestCase: "name": self.name, "description": self.description, "project_name": self.project_name, - "creation_date": str(self.creation_date) + "creation_date": str(self.creation_date), + "url": self.url } def format_http(self, test_project=None): @@ -113,6 +122,7 @@ class TestCase: "name": self.name, "description": self.description, "creation_date": str(self.creation_date), + "url": self.url, } if test_project is not None: res["test_project"] = test_project @@ -127,9 +137,9 @@ class TestResult: self._id = None self.case_name = None self.project_name = None - self.pod_id = None + self.pod_name = None self.installer = None - self.platform_version = None + self.version = None self.description = None self.creation_date = None self.details = None @@ -143,11 +153,13 @@ class TestResult: t = TestResult() t._id = test_result_dict.get('_id') t.case_name = test_result_dict.get('case_name') + t.pod_name = test_result_dict.get('pod_name') t.project_name = test_result_dict.get('project_name') - t.pod_id = test_result_dict.get('pod_id') t.description = test_result_dict.get('description') t.creation_date = str(test_result_dict.get('creation_date')) t.details = test_result_dict.get('details') + t.version = test_result_dict.get('version') + t.installer = test_result_dict.get('installer') return t @@ -155,9 +167,11 @@ class TestResult: return { "case_name": self.case_name, "project_name": self.project_name, - "pod_id": self.pod_id, + "pod_name": self.pod_name, "description": self.description, "creation_date": str(self.creation_date), + "version": self.version, + "installer": self.installer, "details": self.details, } @@ -166,8 +180,10 @@ class TestResult: "_id": str(self._id), "case_name": self.case_name, "project_name": self.project_name, - "pod_id": self.pod_id, + "pod_name": self.pod_name, "description": self.description, "creation_date": str(self.creation_date), + "version": self.version, + "installer": self.installer, "details": self.details, } diff --git a/utils/test/result_collection_api/result_collection_api.py b/utils/test/result_collection_api/result_collection_api.py index c04e0343b..49695772d 100644 --- a/utils/test/result_collection_api/result_collection_api.py +++ b/utils/test/result_collection_api/result_collection_api.py @@ -15,14 +15,12 @@ Pre-requisites: We can launch the API with this file TODOs : - - use POD name instead of id - logging - json args validation with schemes - POST/PUT/DELETE for PODs - POST/PUT/GET/DELETE for installers, platforms (enrich results info) - count cases for GET on test_projects - count results for GET on cases - - provide filtering on requests - include objects - swagger documentation - setup file @@ -48,7 +46,8 @@ args = parser.parse_args() CONF = APIConfig().parse(args.config_file) # connecting to MongoDB server, and choosing database -db = motor.MotorClient(CONF.mongo_url) +client = motor.MotorClient(CONF.mongo_url) +db = client[CONF.mongo_dbname] def make_app(): @@ -61,7 +60,7 @@ def make_app(): # GET /pods => Get all pods # GET /pods/1 => Get details on POD 1 (r"/pods", PodHandler), - (r"/pods/(\d*)", PodHandler), + (r"/pods/([^/]+)", PodHandler), # few examples: # GET /test_projects |