diff options
44 files changed, 1380 insertions, 462 deletions
@@ -21,6 +21,7 @@ Ryota Mibu (NEC, r-mibu@cq.jp.nec.com) Mei Mei (Huawei, meimei@huawei.com) Trevor Bramwell (Linux Foundation, tbramwell@linuxfoundation.org) Serena Feng (ZTE, feng.xiaowei@zte.com.cn) +Yolanda Robla Mota (Red Hat, yroblamo@redhat.com) Link to TSC approval of the project: http://ircbot.wl.linuxfoundation.org/meetings/opnfv-meeting/2015/opnfv-meeting.2015-07-14-14.00.html Link to TSC voting for removal of Victor Laza as committer: http://meetbot.opnfv.org/meetings/opnfv-meeting/2016/opnfv-meeting.2016-02-16-14.59.html diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 89965d77b..643972bd2 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -220,13 +220,13 @@ git-revision: false block: true same-node: true - - trigger-builds: - - project: 'functest-apex-{verify-slave}-suite-{stream}' - predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l2-nofeature-ha - FUNCTEST_SUITE_NAME=healthcheck - block: true - same-node: true +# - trigger-builds: +# - project: 'functest-apex-{verify-slave}-suite-{stream}' +# predefined-parameters: | +# DEPLOY_SCENARIO=os-odl_l2-nofeature-ha +# FUNCTEST_SUITE_NAME=healthcheck +# block: true +# same-node: true - 'apex-workspace-cleanup' - job-template: diff --git a/jjb/armband/armband-project-jobs.yml b/jjb/armband/armband-project-jobs.yml index 10f8d6531..4b2a7b50e 100644 --- a/jjb/armband/armband-project-jobs.yml +++ b/jjb/armband/armband-project-jobs.yml @@ -10,7 +10,6 @@ installer: 'fuel' jobs: - - 'armband-verify-{stream}' - 'armband-{installer}-build-daily-{stream}' stream: @@ -22,48 +21,6 @@ gs-pathname: '/{stream}' - job-template: - name: 'armband-verify-{stream}' - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**|.gitignore' - - builders: - - shell: | - echo "Nothing to verify!" - -- job-template: name: 'armband-{installer}-build-daily-{stream}' concurrent: false diff --git a/jjb/armband/armband-verify-jobs.yml b/jjb/armband/armband-verify-jobs.yml new file mode 100644 index 000000000..90fdd7eeb --- /dev/null +++ b/jjb/armband/armband-verify-jobs.yml @@ -0,0 +1,263 @@ +- project: + name: 'armband-verify-jobs' + + project: 'armband' + + installer: 'fuel' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - colorado: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false +##################################### +# patch verification phases +##################################### + phase: + - 'basic': + slave-label: 'opnfv-build-arm' + - 'build': + slave-label: 'opnfv-build-arm' + - 'deploy-virtual': + slave-label: 'opnfv-build-arm' + - 'smoke-test': + slave-label: 'opnfv-build-arm' +##################################### +# jobs +##################################### + jobs: + - 'armband-verify-{stream}' + - 'armband-verify-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'armband-verify-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'ci/**' + - compare-type: ANT + pattern: 'patches/**' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-arm-defaults' + - 'armband-verify-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'armband-verify-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'armband-verify-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'armband-verify-deploy-virtual-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: smoke-test + condition: SUCCESSFUL + projects: + - name: 'armband-verify-smoke-test-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'armband-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 6 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'armband-verify-deploy-.*' + - 'armband-verify-test-.*' + block-level: 'NODE' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - '{installer}-defaults' + - 'armband-verify-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-macro' +##################################### +# builder macros +##################################### +- builder: + name: 'armband-verify-basic-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" + +- builder: + name: 'armband-verify-build-macro' + builders: + - shell: + !include-raw: ./build.sh + - shell: + !include-raw: ./armband-workspace-cleanup.sh + +- builder: + name: 'armband-verify-deploy-virtual-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" + +- builder: + name: 'armband-verify-smoke-test-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" +##################################### +# parameter macros +##################################### +- parameter: + name: 'armband-verify-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/armband/armband-workspace-cleanup.sh b/jjb/armband/armband-workspace-cleanup.sh new file mode 100755 index 000000000..d8948c7a0 --- /dev/null +++ b/jjb/armband/armband-workspace-cleanup.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +# delete the $WORKSPACE to open some space +/bin/rm -rf $WORKSPACE diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index da882cdfe..eb9113114 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -100,7 +100,7 @@ blocking-jobs: - 'compass-os-.*?-{pod}-daily-.*?' - 'compass-os-.*?-baremetal-daily-.*?' - - 'compass-verify-[^-]*' + - 'compass-verify-[^-]*-[^-]*' block-level: 'NODE' wrappers: diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml index 4d799af8e..3a52e91c9 100644 --- a/jjb/compass4nfv/compass-project-jobs.yml +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -16,145 +16,14 @@ branch: 'stable/{stream}' gs-pathname: '/{stream}' - distro: - - 'trusty': - disabled: false - os-version: 'trusty' - openstack-os-version: '' - - 'centos7': - disabled: false - os-version: 'centos7' - openstack-os-version: '' - jobs: - - 'compass-verify-{distro}-{stream}' - 'compass-build-iso-{stream}' - 'compass-build-ppa-{stream}' - - 'compass-verify-deploy-{distro}-{stream}' - ######################## # job templates ######################## - job-template: - name: 'compass-verify-{distro}-{stream}' - - disabled: false - - concurrent: true - - wrappers: - - timeout: - timeout: 120 - fail: true - - properties: - - throttle: - enabled: true - max-per-node: 1 - option: 'project' - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'compass-verify-[^-]*' - - 'compass-os-.*?-virtual-daily-.*?' - block-level: 'NODE' - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - compass-project-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - '{installer}-defaults' - - '{slave-label}-defaults' - - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' - - triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: '**/*' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**' - - builders: - - trigger-builds: - - project: 'compass-verify-deploy-{distro}-{stream}' - current-parameters: true - predefined-parameters: | - COMPASS_OS_VERSION={os-version} - COMPASS_OS_VERSION_OPTION={openstack-os-version} - same-node: true - block: true - - trigger-builds: - - project: 'functest-{slave-label}-suite-{stream}' - current-parameters: true - predefined-parameters: - FUNCTEST_SUITE_NAME=healthcheck - same-node: true - block: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - -- job-template: - name: 'compass-verify-deploy-{distro}-{stream}' - - concurrent: true - - properties: - - throttle: - enabled: true - max-per-node: 1 - option: 'project' - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'compass-deploy-virtual-daily-.*?' - block-level: 'NODE' - - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - builders: - - shell: - !include-raw-escape: ./compass-build.sh - - shell: - !include-raw-escape: ./compass-deploy.sh - - publishers: - - archive: - artifacts: 'ansible.log' - allow-empty: 'true' - fingerprint: true - -- job-template: name: 'compass-build-iso-{stream}' concurrent: true diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml new file mode 100644 index 000000000..82e3dfdf2 --- /dev/null +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -0,0 +1,251 @@ +- project: + name: 'compass-verify-jobs' + + project: 'compass4nfv' + + installer: 'compass' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - colorado: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: true + + distro: + - 'trusty': + disabled: false + os-version: 'trusty' + openstack-os-version: '' + - 'centos7': + disabled: false + os-version: 'centos7' + openstack-os-version: '' +##################################### +# patch verification phases +##################################### + phase: + - 'basic' + - 'deploy-virtual' +##################################### +# jobs +##################################### + jobs: + - 'compass-verify-{distro}-{stream}' + - 'compass-verify-{phase}-{distro}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'compass-verify-{distro}-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'compass-verify-[^-]*-[^-]*' + - 'compass-os-.*?-virtual-daily-.*?' + block-level: 'NODE' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 120 + fail: true + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'compass-virtual-defaults' + - '{installer}-defaults' + - 'compass-verify-defaults': + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'compass-verify-basic-{stream}' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'compass-verify-deploy-virtual-{distro}-{stream}' + current-parameters: true + predefined-parameters: | + COMPASS_OS_VERSION={os-version} + COMPASS_OS_VERSION_OPTION={openstack-os-version} + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: smoke-test + condition: SUCCESSFUL + projects: + - name: 'functest-compass-virtual-suite-{stream}' + current-parameters: true + predefined-parameters: + FUNCTEST_SUITE_NAME=healthcheck + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + +- job-template: + name: 'compass-verify-{phase}-{distro}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'compass-os-.*?-virtual-daily-.*?' + - 'compass-verify-deploy-.*' + - 'functest-compass-virtual.*' + block-level: 'NODE' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 120 + fail: true + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-macro' + + publishers: + - archive: + artifacts: 'ansible.log' + allow-empty: 'true' + fingerprint: true +##################################### +# builder macros +##################################### +- builder: + name: 'compass4nfv-verify-basic-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" + +- builder: + name: 'compass4nfv-verify-deploy-virtual-macro' + builders: + - shell: + !include-raw: ./compass-build.sh + - shell: + !include-raw: ./compass-deploy.sh +##################################### +# parameter macros +##################################### +- parameter: + name: 'compass-verify-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: "$HOME/opnfv/cache/$PROJECT{gs-pathname}" + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: '$GS_BASE{gs-pathname}' + description: "URL to Google Storage." + - string: + name: PPA_REPO + default: "http://205.177.226.237:9999{gs-pathname}" + - string: + name: PPA_CACHE + default: "$WORKSPACE/work/repo/" + - choice: + name: COMPASS_OPENSTACK_VERSION + choices: + - 'mitaka' + - 'liberty' + - choice: + name: COMPASS_OS_VERSION + choices: + - 'trusty' + - 'centos7' diff --git a/jjb/daisy4nfv/daisy4nfv-basic.sh b/jjb/daisy4nfv/daisy4nfv-basic.sh index 87f5482e0..04b9b7bfa 100755 --- a/jjb/daisy4nfv/daisy4nfv-basic.sh +++ b/jjb/daisy4nfv/daisy4nfv-basic.sh @@ -4,3 +4,4 @@ echo "--------------------------------------------------------" echo "This is diasy4nfv basic job!" echo "--------------------------------------------------------" +sudo rm -rf /home/jenkins-ci/opnfv/slave_root/workspace/daisy4nfv-verify-build-master/* diff --git a/jjb/daisy4nfv/daisy4nfv-build.sh b/jjb/daisy4nfv/daisy4nfv-build.sh index ec11db587..eb29fed72 100755 --- a/jjb/daisy4nfv/daisy4nfv-build.sh +++ b/jjb/daisy4nfv/daisy4nfv-build.sh @@ -4,11 +4,31 @@ echo "--------------------------------------------------------" echo "This is diasy4nfv build job!" echo "--------------------------------------------------------" +# set OPNFV_ARTIFACT_VERSION +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Building Daisy4nfv ISO for a merged change" + export OPNFV_ARTIFACT_VERSION="gerrit-$GERRIT_CHANGE_NUMBER" +else + export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +fi + # build output directory OUTPUT_DIR=$WORKSPACE/build_output mkdir -p $OUTPUT_DIR # start the build cd $WORKSPACE -./ci/build.sh $OUTPUT_DIR +./ci/build.sh $OUTPUT_DIR $OPNFV_ARTIFACT_VERSION + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin" + echo "OPNFV_BUILD_URL=$BUILD_URL" +) > $WORKSPACE/opnfv.properties +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh new file mode 100755 index 000000000..7ac76a5eb --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh @@ -0,0 +1,48 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o pipefail + +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties" + # get the properties file for the Daisy4nfv BIN built for a merged change + curl -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties +else + # get the latest.properties file in order to get info regarding latest artifact + echo "Downloading http://$GS_URL/latest.properties" + curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties +fi + +# check if we got the file +[[ -f latest.properties ]] || exit 1 + +# source the file so we get artifact metadata +source latest.properties + +# echo the info about artifact that is used during the deployment +OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/} +echo "Using $OPNFV_ARTIFACT for deployment" + +# log info to console +echo "Downloading the $INSTALLER_TYPE artifact using URL http://$OPNFV_ARTIFACT_URL" +echo "This could take some time..." +echo "--------------------------------------------------------" +echo + +# download the file +curl -s -o $WORKSPACE/opnfv.bin http://$OPNFV_ARTIFACT_URL > gsutil.bin.log 2>&1 + +# list the file +ls -al $WORKSPACE/opnfv.bin + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml new file mode 100644 index 000000000..f8c0258a6 --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml @@ -0,0 +1,270 @@ +- project: + name: 'daisy4nfv-merge-jobs' + + project: 'daisy' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# patch merge phases +##################################### + phase: + - 'basic': + slave-label: 'opnfv-build-centos' + - 'build': + slave-label: 'opnfv-build-centos' + - 'deploy-virtual': + slave-label: 'opnfv-build-centos' + - 'smoke-test': + slave-label: 'opnfv-build-centos' + - 'promote': + slave-label: 'opnfv-build-centos' +##################################### +# jobs +##################################### + jobs: + - 'daisy4nfv-merge-{stream}' + - 'daisy4nfv-merge-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'daisy4nfv-merge-{stream}' + + project-type: multijob + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'daisy4nfv-merge-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-deploy-virtual-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: smoke-test + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-smoke-test-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: promote + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-promote-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'daisy4nfv-merge-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 6 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'daisy4nfv-merge-deploy-.*' + - 'daisy4nfv-merge-test-.*' + block-level: 'NODE' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - 'daisy4nfv-merge-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-merge-{phase}-macro' +##################################### +# builder macros +##################################### +- builder: + name: 'daisy-merge-basic-macro' + builders: + - shell: + !include-raw: ./daisy4nfv-basic.sh + +- builder: + name: 'daisy-merge-build-macro' + builders: + - shell: + !include-raw: + - ./daisy4nfv-build.sh + - ./daisy4nfv-upload-artifact.sh + - ./daisy4nfv-workspace-cleanup.sh + +- builder: + name: 'daisy-merge-deploy-virtual-macro' + builders: + - shell: + !include-raw: + - ./daisy4nfv-download-artifact.sh + - ./daisy4nfv-virtual-deploy.sh + - ./daisy4nfv-workspace-cleanup.sh + +- builder: + name: 'daisy-merge-smoke-test-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" + +- builder: + name: 'daisy-merge-promote-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" +##################################### +# parameter macros +##################################### +- parameter: + name: 'daisy4nfv-merge-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh b/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh new file mode 100755 index 000000000..6b0aec54f --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh @@ -0,0 +1,94 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o pipefail + +# check if we built something +if [ -f $WORKSPACE/.noupload ]; then + echo "Nothing new to upload. Exiting." + /bin/rm -f $WORKSPACE/.noupload + exit 0 +fi + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +importkey () { +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi +} + +signbin () { +gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin + +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin.sig +echo "BIN signature Upload Complete!" +} + +uploadbin () { +# log info to console +echo "Uploading $INSTALLER_TYPE artifact. This could take some time..." +echo + +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > gsutil.bin.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +elif [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Uploaded Daisy4nfv BIN for a merged change" +fi + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.bin.log on the machine where this build is done." + exit 1 +fi + +echo "Done!" +echo +echo "--------------------------------------------------------" +echo +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin" +echo +echo "--------------------------------------------------------" +echo +} + +importkey +signbin +uploadbin diff --git a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml index 7c47d9f69..d893b1485 100644 --- a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml @@ -15,7 +15,7 @@ ##################################### phase: - 'basic': - slave-label: 'opnfv-build' + slave-label: 'opnfv-build-centos' - 'build': slave-label: 'opnfv-build-centos' - 'deploy-virtual': @@ -101,6 +101,11 @@ projects: - name: 'daisy4nfv-verify-basic-{stream}' current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: false kill-phase-on: FAILURE abort-all-job: true @@ -110,6 +115,11 @@ projects: - name: 'daisy4nfv-verify-build-{stream}' current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: false kill-phase-on: FAILURE abort-all-job: true @@ -119,6 +129,11 @@ projects: - name: 'daisy4nfv-verify-deploy-virtual-{stream}' current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: false kill-phase-on: FAILURE abort-all-job: true @@ -128,6 +143,11 @@ projects: - name: 'daisy4nfv-verify-smoke-test-{stream}' current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: false kill-phase-on: FAILURE abort-all-job: true diff --git a/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh b/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh new file mode 100755 index 000000000..26f7e9a01 --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +# delete the $WORKSPACE to open some space +/bin/rm -rf $WORKSPACE diff --git a/jjb/fuel/fuel-upload-artifact.sh b/jjb/fuel/fuel-upload-artifact.sh index 1ccd3282c..ca4ba00b0 100755 --- a/jjb/fuel/fuel-upload-artifact.sh +++ b/jjb/fuel/fuel-upload-artifact.sh @@ -108,13 +108,11 @@ echo nfsstore -if [[ ! "$JOB_NAME" =~ merge ]]; then +if [[ "$JOB_NAME" =~ merge ]]; then + uploadiso +elif [[ "$JOB_NAME" =~ build ]]; then importkey signiso uploadiso fi -if [[ ! "$JOB_NAME" =~ verify ]]; then - uploadiso -fi - diff --git a/jjb/functest/functest-loop.sh b/jjb/functest/functest-loop.sh index 73856235f..4528c00d1 100755 --- a/jjb/functest/functest-loop.sh +++ b/jjb/functest/functest-loop.sh @@ -5,8 +5,10 @@ branch=${GIT_BRANCH##*/} [[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+="-r" if [[ ${branch} == *"brahmaputra"* ]]; then cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh -s ${flags}" -else +elif [[ ${branch} == *"colorado"* ]]; then cmd="python ${FUNCTEST_REPO_DIR}/ci/run_tests.py -t all ${flags}" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/run_tests.py -t all ${flags}" fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd @@ -15,4 +17,4 @@ ret_value=$? ret_val_file="${HOME}/opnfv/functest/results/${branch}/return_value" echo ${ret_value}>${ret_val_file} -exit 0
\ No newline at end of file +exit 0 diff --git a/jjb/functest/functest-project-jobs.yml b/jjb/functest/functest-project-jobs.yml index a9845459f..236b95d92 100644 --- a/jjb/functest/functest-project-jobs.yml +++ b/jjb/functest/functest-project-jobs.yml @@ -18,7 +18,7 @@ - colorado: branch: 'stable/{stream}' gs-pathname: '/{stream}' - disabled: false + disabled: true - job-template: name: 'functest-verify-{stream}' @@ -61,5 +61,33 @@ pattern: 'docs/**|.gitignore' builders: + - functest-unit-tests-and-docs-build + + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + +################################ +# job builders +################################ + +- builder: + name: functest-unit-tests-and-docs-build + builders: - shell: | - echo "Nothing to verify!" + $WORKSPACE/run_unit_tests.sh diff --git a/jjb/functest/functest-suite.sh b/jjb/functest/functest-suite.sh index 7e9fa09db..df286569f 100755 --- a/jjb/functest/functest-suite.sh +++ b/jjb/functest/functest-suite.sh @@ -5,8 +5,10 @@ branch=${GIT_BRANCH##*/} echo "Functest: run $FUNCTEST_SUITE_NAME on branch ${branch}" if [[ ${branch} == *"brahmaputra"* ]]; then cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME" -else +elif [[ ${branch} == *"colorado"* ]]; then cmd="python ${FUNCTEST_REPO_DIR}/ci/run_tests.py -t $FUNCTEST_SUITE_NAME" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/run_tests.py -t $FUNCTEST_SUITE_NAME" fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd diff --git a/jjb/functest/set-functest-env.sh b/jjb/functest/set-functest-env.sh index 1c77702eb..bc30ffaf3 100755 --- a/jjb/functest/set-functest-env.sh +++ b/jjb/functest/set-functest-env.sh @@ -82,8 +82,10 @@ if [ $(docker ps | grep "opnfv/functest:${DOCKER_TAG}" | wc -l) == 0 ]; then fi if [[ ${branch} == *"brahmaputra"* ]]; then cmd="${FUNCTEST_REPO_DIR}/docker/prepare_env.sh" -else +elif [[ ${branch} == *"colorado"* ]]; then cmd="python ${FUNCTEST_REPO_DIR}/ci/prepare_env.py start" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/prepare_env.py start" fi echo "Executing command inside the docker: ${cmd}" docker exec ${container_id} ${cmd} diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml index d15bf7426..751aa0c4a 100644 --- a/jjb/infra/bifrost-verify-jobs.yml +++ b/jjb/infra/bifrost-verify-jobs.yml @@ -28,7 +28,7 @@ - 'centos7': disabled: false dib-os-release: '7' - dib-os-element: 'centos-minimal' + dib-os-element: 'centos7' dib-os-packages: 'openssh-server,vim,less,bridge-utils,iputils,rsyslog,curl' - 'suse': disabled: true diff --git a/jjb/infra/bifrost-verify.sh b/jjb/infra/bifrost-verify.sh index f7920a36f..ded4ed463 100755 --- a/jjb/infra/bifrost-verify.sh +++ b/jjb/infra/bifrost-verify.sh @@ -46,6 +46,13 @@ sudo git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD # combine opnfv and upstream scripts/playbooks sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/ +# place bridge creation file on the right path +sudo mkdir -p /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt +sudo cp /opt/puppet-infracloud/templates/bifrost/create_bridge.py.erb /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py + +# replace bridge name +sudo sed -i s/"<%= @bridge_name -%>"/br_opnfv/g /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py + # cleanup remnants of previous deployment cd /opt/bifrost sudo -E ./scripts/destroy-env.sh diff --git a/jjb/kvmfornfv/kvmfornfv-test.sh b/jjb/kvmfornfv/kvmfornfv-test.sh index 06377ac4f..f677f470d 100755 --- a/jjb/kvmfornfv/kvmfornfv-test.sh +++ b/jjb/kvmfornfv/kvmfornfv-test.sh @@ -18,7 +18,7 @@ case "$JOB_TYPE" in verify|daily) #start the test cd $WORKSPACE - ./ci/test_kvmfornfv.sh $JOB_TYPE + ./ci/test_kvmfornfv.sh $JOB_TYPE $TEST_NAME ;; *) echo "Test is not enabled for $JOB_TYPE jobs" diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml index b6a55fe6a..2c8446e07 100644 --- a/jjb/kvmfornfv/kvmfornfv.yml +++ b/jjb/kvmfornfv/kvmfornfv.yml @@ -23,12 +23,22 @@ ##################################### # patch verification phases ##################################### + testname: + - 'Idle_Idle': + trigger_time: '@midnight' + - 'Stress_Idle': + trigger_time: '@midnight+3hours' + - 'Packet_Forwarding': + trigger_time: '@midnight+6hours' +##################################### +# patch verification phases +##################################### jobs: - 'kvmfornfv-verify-{stream}' - 'kvmfornfv-verify-{phase}-{stream}' - 'kvmfornfv-merge-{stream}' - - 'kvmfornfv-daily-{stream}' - - 'kvmfornfv-daily-{phase}-{stream}' + - 'kvmfornfv-{testname}-daily-{stream}' + - 'kvmfornfv-{testname}-daily-{phase}-{stream}' ##################################### # job templates ##################################### @@ -103,6 +113,7 @@ kill-phase-on: FAILURE abort-all-job: true + - job-template: name: 'kvmfornfv-verify-{phase}-{stream}' @@ -175,7 +186,7 @@ !include-raw: ./kvmfornfv-build.sh - job-template: - name: 'kvmfornfv-daily-{stream}' + name: 'kvmfornfv-{testname}-daily-{stream}' project-type: multijob @@ -199,7 +210,7 @@ choosing-strategy: 'default' triggers: - - timed: '@midnight' + - timed: '{trigger_time}' builders: - description-setter: @@ -208,17 +219,59 @@ name: build condition: SUCCESSFUL projects: - - name: 'kvmfornfv-daily-build-{stream}' + - name: 'kvmfornfv-Idle_Idle-daily-build-{stream}' + current-parameters: false + node-parameters: false + git-revision: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'kvmfornfv-Stress_Idle-daily-build-{stream}' + current-parameters: false + node-parameters: false + git-revision: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'kvmfornfv-Packet_Forwarding-daily-build-{stream}' current-parameters: false node-parameters: false git-revision: true kill-phase-on: FAILURE abort-all-job: true + + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'kvmfornfv-Idle_Idle-daily-test-{stream}' + current-parameters: false + node-parameters: false + git-revision: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: name: test condition: SUCCESSFUL projects: - - name: 'kvmfornfv-daily-test-{stream}' + - name: 'kvmfornfv-Stress_Idle-daily-test-{stream}' + current-parameters: false + node-parameters: false + git-revision: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'kvmfornfv-Packet_Forwarding-daily-build-{stream}' current-parameters: false node-parameters: false git-revision: true @@ -227,7 +280,7 @@ - job-template: - name: 'kvmfornfv-daily-{phase}-{stream}' + name: 'kvmfornfv-{testname}-daily-{phase}-{stream}' disabled: '{obj:disabled}' @@ -258,7 +311,31 @@ builders: - description-setter: description: "Built on $NODE_NAME" - - '{project}-daily-{phase}-macro' + - '{project}-{testname}-daily-{phase}-macro' +######################## +# parameter macros +######################## +- parameter: + name: 'kvmfornfv-Idle_Idle-daily-parameter' + parameters: + - string: + name: TEST_NAME + default: 'idle_idle' + description: "Daily job to run cyclictest without applying any stress" +- parameter: + name: 'kvmfornfv-Stress_Idle-daily-parameter' + parameters: + - string: + name: TEST_NAME + default: 'stress_idle' + description: "Daily job to run cyclictest with stress applied" +- parameter: + name: 'kvmfornfv-Packet_Forwarding-daily-parameter' + parameters: + - string: + name: TEST_NAME + default: 'packet_forward' + description: "Daily job to run packet forwarding test cases" ##################################### # builder macros ##################################### @@ -277,14 +354,42 @@ - shell: !include-raw: ./kvmfornfv-test.sh - builder: - name: 'kvmfornfv-daily-build-macro' + name: 'kvmfornfv-Idle_Idle-daily-build-macro' + builders: + - shell: + !include-raw: ./kvmfornfv-build.sh + - shell: + !include-raw: ./kvmfornfv-upload-artifact.sh +- builder: + name: 'kvmfornfv-Stress_Idle-daily-build-macro' + builders: + - shell: + !include-raw: ./kvmfornfv-build.sh + - shell: + !include-raw: ./kvmfornfv-upload-artifact.sh +- builder: + name: 'kvmfornfv-Packet_Forwarding-daily-build-macro' builders: - shell: !include-raw: ./kvmfornfv-build.sh - shell: !include-raw: ./kvmfornfv-upload-artifact.sh - builder: - name: 'kvmfornfv-daily-test-macro' + name: 'kvmfornfv-Idle_Idle-daily-test-macro' + builders: + - shell: + !include-raw: ./kvmfornfv-download-artifact.sh + - shell: + !include-raw: ./kvmfornfv-test.sh +- builder: + name: 'kvmfornfv-Stress_Idle-daily-test-macro' + builders: + - shell: + !include-raw: ./kvmfornfv-download-artifact.sh + - shell: + !include-raw: ./kvmfornfv-test.sh +- builder: + name: 'kvmfornfv-Packet_Forwarding-daily-test-macro' builders: - shell: !include-raw: ./kvmfornfv-download-artifact.sh diff --git a/jjb/opnfv/artifact-cleanup.yml b/jjb/opnfv/artifact-cleanup.yml index b0f819145..1c609a4b2 100644 --- a/jjb/opnfv/artifact-cleanup.yml +++ b/jjb/opnfv/artifact-cleanup.yml @@ -1,10 +1,10 @@ - project: - name: artifact-cleanup + name: releng-artifact-cleanup project: 'releng' jobs: - - 'artifact-cleanup-daily-{stream}' + - 'releng-artifact-cleanup-daily-{stream}' stream: - master: @@ -13,7 +13,7 @@ - job-template: - name: 'artifact-cleanup-daily-{stream}' + name: 'releng-artifact-cleanup-daily-{stream}' # Job template for daily builders # diff --git a/jjb/opnfv/opnfv-lint.yml b/jjb/opnfv/opnfv-lint.yml index f90f95dc2..6860dd037 100644 --- a/jjb/opnfv/opnfv-lint.yml +++ b/jjb/opnfv/opnfv-lint.yml @@ -55,7 +55,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/opnfv/test-sign.yml b/jjb/opnfv/test-sign.yml deleted file mode 100644 index b27d75777..000000000 --- a/jjb/opnfv/test-sign.yml +++ /dev/null @@ -1,42 +0,0 @@ -- project: - name: test-sign - - project: 'releng' - - jobs: - - 'test-sign-daily-{stream}' - - stream: - - master: - branch: '{stream}' - gs-pathname: '' - - -- job-template: - name: 'test-sign-daily-{stream}' - - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: master - - disabled: false - - parameters: - - project-parameter: - project: '{project}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - triggers: - - timed: 'H H * * *' - - builders: - - shell: | - $WORKSPACE/utils/test-sign-artifact.sh diff --git a/jjb/qtip/qtip-ci-jobs.yml b/jjb/qtip/qtip-ci-jobs.yml index d0d6b47a3..cca8cee4c 100644 --- a/jjb/qtip/qtip-ci-jobs.yml +++ b/jjb/qtip/qtip-ci-jobs.yml @@ -40,6 +40,10 @@ installer: fuel auto-trigger-name: 'qtip-daily-zte-pod2-trigger' <<: *master + - zte-pod3: + installer: fuel + auto-trigger-name: 'qtip-daily-zte-pod3-trigger' + <<: *master #-------------------------------- jobs: @@ -120,5 +124,9 @@ - trigger: name: 'qtip-daily-zte-pod2-trigger' triggers: - - timed: '0 5 * * *' + - timed: '0 7 * * *' +- trigger: + name: 'qtip-daily-zte-pod3-trigger' + triggers: + - timed: '0 1 * * *' diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index d2dc1d1ec..3afd355e6 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -171,12 +171,18 @@ echo "########################" echo + echo "gs_path="$GS_URL/docs"" + echo "local_path="upload/docs"" + gs_path="$GS_URL/docs" local_path="upload/docs" mkdir -p upload mv docs_output "$local_path" - gsutil -m cp -r "$local_path" "gs://$GS_URL" + ls "$local_path" + + echo "gsutil -m cp -r "$local_path"/* "gs://$gs_path"" + gsutil -m cp -r "$local_path"/* "gs://$gs_path" gsutil -m setmeta \ -h "Content-Type:text/html" \ diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index 2d88449e8..ccfe11e9a 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -1,14 +1,14 @@ - project: - name: builder-jobs + name: releng-builder-jobs jobs: - - 'builder-verify-jjb' - - 'builder-merge' - - 'artifacts-api' + - 'releng-verify-jjb' + - 'releng-merge-jjb' + - 'releng-generate-artifacts-api' project: 'releng' - job-template: - name: builder-verify-jjb + name: releng-verify-jjb parameters: - project-parameter: @@ -57,7 +57,7 @@ artifacts: 'job_output/*' - job-template: - name: 'builder-merge' + name: 'releng-merge-jjb' # builder-merge job to run JJB update # @@ -101,7 +101,7 @@ jenkins-jobs update -r --delete-old jjb/ - job-template: - name: 'artifacts-api' + name: 'releng-generate-artifacts-api' # Generate and upload the JSON file to used for artifacts site diff --git a/jjb/vswitchperf/vswitchperf.yml b/jjb/vswitchperf/vswitchperf.yml index 233cf08bc..3f7f6bf2d 100644 --- a/jjb/vswitchperf/vswitchperf.yml +++ b/jjb/vswitchperf/vswitchperf.yml @@ -14,10 +14,12 @@ branch: '{stream}' gs-pathname: '' disabled: false + slave-label: 'opnfv-build-ubuntu' - colorado: branch: 'stable/{stream}' gs-pathname: '/{stream}' disabled: false + slave-label: 'intel-pod3' - job-template: @@ -72,7 +74,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - '{slave-label}-defaults' scm: - gerrit-trigger-scm: @@ -134,7 +136,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' + - '{slave-label}-defaults' scm: - gerrit-trigger-scm: diff --git a/jjb/yardstick/yardstick-ci-jobs.yml b/jjb/yardstick/yardstick-ci-jobs.yml index c10daabe0..9d80e42ae 100644 --- a/jjb/yardstick/yardstick-ci-jobs.yml +++ b/jjb/yardstick/yardstick-ci-jobs.yml @@ -315,7 +315,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-armband-baremetal' @@ -329,7 +329,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-joid-baremetal' @@ -343,14 +343,14 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-intel-pod8' parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: name: 'yardstick-params-lf-pod1' @@ -445,7 +445,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -453,7 +453,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -461,7 +461,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -469,7 +469,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' ####################### diff --git a/prototypes/bifrost/README.md b/prototypes/bifrost/README.md index df34f9c87..0ba49d46d 100644 --- a/prototypes/bifrost/README.md +++ b/prototypes/bifrost/README.md @@ -21,14 +21,17 @@ Please follow that steps: sudo cp -R /opt/releng/prototypes/bifrost/* /opt/bifrost/ -5. If you are on a RHEL/CentOS box, ensure that selinux is disabled +5. Copy /opt/puppet-infracloud/templates/bifrost/create_bridge.py.erb to /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py, + and replace tag <%= @bridge_name -%> with br_opnfv -6. Run destroy script if you need to cleanup previous environment:: +6. If you are on a RHEL/CentOS box, ensure that selinux is disabled + +7. Run destroy script if you need to cleanup previous environment:: cd /opt/bifrost sudo ./scripts/destroy-env.sh -7. Run deployment script to spin up 3 vms with bifrost: jumphost, controller and compute:: +8. Run deployment script to spin up 3 vms with bifrost: jumphost, controller and compute:: cd /opt/bifrost sudo ./scripts/test-bifrost-deployment.sh @@ -41,10 +44,10 @@ It is likely that the script will show some errors due to timeout. Please ignore And wait until all the vms are in **active** Provisioning State. -8. Check the IPs assigned to each of the VMS. You can check it by looking at inventory: +9. Check the IPs assigned to each of the VMS. You can check it by looking at inventory: cat /tmp/baremetal.csv -9. You can enter into the vms with devuser login/pass: +10. You can enter into the vms with devuser login/pass: ssh devuser@192.168.122.2 diff --git a/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml b/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml index d650f1056..a0bc28506 100644 --- a/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml +++ b/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml @@ -67,18 +67,18 @@ - role: ironic-enroll-dynamic - { role: ironic-inspect-node, when: inspect_nodes | default('false') | bool == true } - hosts: baremetal - name: "Create configuration drive files" + name: "Create configuration drive files and deploy machines" + vars: + multinode_testing: "{{ inventory_dhcp | bool == true }}" become: no connection: local roles: - role: bifrost-configdrives-dynamic + - role: bifrost-deploy-nodes-dynamic - hosts: baremetal - vars: - multinode_testing: "{{ inventory_dhcp | bool == true }}" name: "Deploy machines." become: no connection: local serial: 1 roles: - - role: bifrost-deploy-nodes-dynamic - role: bifrost-prepare-for-test-dynamic diff --git a/prototypes/bifrost/scripts/test-bifrost-deployment.sh b/prototypes/bifrost/scripts/test-bifrost-deployment.sh index 773697efe..030faedcf 100755 --- a/prototypes/bifrost/scripts/test-bifrost-deployment.sh +++ b/prototypes/bifrost/scripts/test-bifrost-deployment.sh @@ -58,7 +58,7 @@ export ELEMENTS_PATH=/usr/share/diskimage-builder/elements:/opt/puppet-infraclou export DIB_DEV_USER_PWDLESS_SUDO=yes export DIB_DEV_USER_PASSWORD=devuser -# settings for distro: trusty/ubuntu-minimal, 7/centos-minimal +# settings for distro: trusty/ubuntu-minimal, 7/centos7 export DIB_OS_RELEASE=${DIB_OS_RELEASE:-trusty} export DIB_OS_ELEMENT=${DIB_OS_ELEMENT:-ubuntu-minimal} diff --git a/prototypes/puppet-infracloud/deploy_on_baremetal.md b/prototypes/puppet-infracloud/deploy_on_baremetal.md index 334dff4d2..2bd0a5303 100644 --- a/prototypes/puppet-infracloud/deploy_on_baremetal.md +++ b/prototypes/puppet-infracloud/deploy_on_baremetal.md @@ -26,6 +26,7 @@ On the same bifrost VM, follow these steps: 1. Source bifrost env vars: source /opt/stack/bifrost/env-vars 2. Export baremetal servers inventory: export BIFROST_INVENTORY-SOURCE=/opt/stack/baremetal.json + 3. Change active directory: cd /opt/stack/bifrost/playbooks 3. Enroll the servers: ansible-playbook -vvv -i inventory/bifrost_inventory.py enroll-dynamic.yaml -e @/etc/bifrost/bifrost_global_vars 4. Deploy the servers: ansible-playbook -vvv -i inventory/bifrost_inventory.py deploy-dynamic.yaml -e @/etc/bifrost/bifrost_global_vars 5. Wait until they are on **active** state, check it with: ironic node-list diff --git a/prototypes/puppet-infracloud/modules/opnfv/manifests/server.pp b/prototypes/puppet-infracloud/modules/opnfv/manifests/server.pp index 6b608a7a7..fc9bf7163 100644 --- a/prototypes/puppet-infracloud/modules/opnfv/manifests/server.pp +++ b/prototypes/puppet-infracloud/modules/opnfv/manifests/server.pp @@ -22,7 +22,7 @@ class opnfv::server ( class { 'iptables': public_tcp_ports => $iptables_public_tcp_ports, - public_udp_ports => $all_udp, + public_udp_ports => $iptables_public_udp_ports, rules4 => $iptables_rules4, rules6 => $iptables_rules6, } diff --git a/utils/installer-adapter/ApexAdapter.py b/utils/installer-adapter/ApexAdapter.py index bf451f3d2..17a27b10a 100644 --- a/utils/installer-adapter/ApexAdapter.py +++ b/utils/installer-adapter/ApexAdapter.py @@ -8,9 +8,6 @@ ############################################################################## -from SSHUtils import SSH_Connection - - class ApexAdapter: def __init__(self, installer_ip): @@ -32,4 +29,4 @@ class ApexAdapter: pass def get_file_from_controller(self, origin, target, ip=None, options=None): - pass
\ No newline at end of file + pass diff --git a/utils/installer-adapter/CompassAdapter.py b/utils/installer-adapter/CompassAdapter.py index b40a8d788..47cbc646d 100644 --- a/utils/installer-adapter/CompassAdapter.py +++ b/utils/installer-adapter/CompassAdapter.py @@ -8,9 +8,6 @@ ############################################################################## -from SSHUtils import SSH_Connection - - class CompassAdapter: def __init__(self, installer_ip): @@ -32,4 +29,4 @@ class CompassAdapter: pass def get_file_from_controller(self, origin, target, ip=None, options=None): - pass
\ No newline at end of file + pass diff --git a/utils/installer-adapter/FuelAdapter.py b/utils/installer-adapter/FuelAdapter.py index 15f0e929f..672fd5175 100644 --- a/utils/installer-adapter/FuelAdapter.py +++ b/utils/installer-adapter/FuelAdapter.py @@ -1,14 +1,14 @@ ############################################################################## # Copyright (c) 2016 Ericsson AB and others. # Author: Jose Lausuch (jose.lausuch@ericsson.com) +# George Paraskevopoulos (geopar@intracom-telecom.com) # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -from SSHUtils import SSH_Connection +import SSHUtils as ssh_utils import RelengLogger as rl @@ -16,25 +16,30 @@ class FuelAdapter: def __init__(self, installer_ip, user="root", password="r00tme"): self.installer_ip = installer_ip - self.user = user - self.password = password - self.connection = SSH_Connection( - installer_ip, self.user, self.password, use_system_keys=False) + self.installer_user = user + self.installer_password = password + self.installer_connection = ssh_utils.get_ssh_client( + installer_ip, + self.installer_user, + password=self.installer_password) self.logger = rl.Logger("Handler").getLogger() - def runcmd_fuel_nodes(self): - output, error = self.connection.run_remote_cmd('fuel nodes') + def runcmd_fuel_installer(self, cmd): + _, stdout, stderr = (self + .installer_connection + .exec_command(cmd)) + error = stderr.readlines() if len(error) > 0: - self.logger.error("error %s" % error) + self.logger.error("error %s" % ''.join(error)) return error + output = ''.join(stdout.readlines()) return output + def runcmd_fuel_nodes(self): + return self.runcmd_fuel_installer('fuel nodes') + def runcmd_fuel_env(self): - output, error = self.connection.run_remote_cmd('fuel env') - if len(error) > 0: - self.logger.error("error %s" % error) - return error - return output + return self.runcmd_fuel_installer('fuel env') def get_clusters(self): environments = [] @@ -183,8 +188,11 @@ class FuelAdapter: def get_file_from_installer(self, remote_path, local_path, options=None): self.logger.debug("Fetching %s from %s" % (remote_path, self.installer_ip)) - if self.connection.scp_get(local_path, remote_path) != 0: - self.logger.error("SCP failed to retrieve the file.") + get_file_result = ssh_utils.get_file(self.installer_connection, + remote_path, + local_path) + if get_file_result is None: + self.logger.error("SFTP failed to retrieve the file.") return 1 self.logger.info("%s successfully copied from Fuel to %s" % (remote_path, local_path)) @@ -193,6 +201,7 @@ class FuelAdapter: remote_path, local_path, ip=None, + user='root', options=None): if ip is None: controllers = self.get_controller_ips(options=options) @@ -204,16 +213,24 @@ class FuelAdapter: else: target_ip = ip - fuel_dir = '/root/scp/' - cmd = 'mkdir -p %s;rsync -Rav %s:%s %s' % ( - fuel_dir, target_ip, remote_path, fuel_dir) - self.logger.info("Copying %s from %s to Fuel..." % - (remote_path, target_ip)) - output, error = self.connection.run_remote_cmd(cmd) - self.logger.debug("Copying files from Fuel to %s..." % local_path) - self.get_file_from_installer( - fuel_dir + remote_path, local_path, options) - cmd = 'rm -r %s' % fuel_dir - output, error = self.connection.run_remote_cmd(cmd) + installer_jumphost = { + 'ip': self.installer_ip, + 'username': self.installer_user, + 'password': self.installer_password + } + controller_conn = ssh_utils.get_ssh_client( + target_ip, + user, + jumphost=installer_jumphost) + + self.logger.debug("Fetching %s from %s" % + (remote_path, target_ip)) + + get_file_result = ssh_utils.get_file(controller_conn, + remote_path, + local_path) + if get_file_result is None: + self.logger.error("SFTP failed to retrieve the file.") + return 1 self.logger.info("%s successfully copied from %s to %s" % (remote_path, target_ip, local_path)) diff --git a/utils/installer-adapter/JoidAdapter.py b/utils/installer-adapter/JoidAdapter.py index e78ca0fae..be8c2ebac 100644 --- a/utils/installer-adapter/JoidAdapter.py +++ b/utils/installer-adapter/JoidAdapter.py @@ -8,9 +8,6 @@ ############################################################################## -from SSHUtils import SSH_Connection - - class JoidAdapter: def __init__(self, installer_ip): @@ -32,4 +29,4 @@ class JoidAdapter: pass def get_file_from_controller(self, origin, target, ip=None, options=None): - pass
\ No newline at end of file + pass diff --git a/utils/installer-adapter/RelengLogger.py b/utils/installer-adapter/RelengLogger.py index b38e78095..6fa4ef2e2 100644 --- a/utils/installer-adapter/RelengLogger.py +++ b/utils/installer-adapter/RelengLogger.py @@ -22,7 +22,6 @@ # logger.debug("message to be shown with - DEBUG -") import logging -import os class Logger: diff --git a/utils/installer-adapter/SSHUtils.py b/utils/installer-adapter/SSHUtils.py index 9c92a3be1..c93888694 100644 --- a/utils/installer-adapter/SSHUtils.py +++ b/utils/installer-adapter/SSHUtils.py @@ -1,6 +1,7 @@ ############################################################################## # Copyright (c) 2015 Ericsson AB and others. -# Author: Jose Lausuch (jose.lausuch@ericsson.com) +# Authors: George Paraskevopoulos (geopar@intracom-telecom.com) +# Jose Lausuch (jose.lausuch@ericsson.com) # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -9,122 +10,111 @@ import paramiko -from scp import SCPClient -import time import RelengLogger as rl +import os +logger = rl.Logger('SSHUtils').getLogger() -class SSH_Connection: - - def __init__(self, - host, - user, - password, - use_system_keys=True, - private_key=None, - use_proxy=False, - proxy_host=None, - proxy_user=None, - proxy_password=None, - timeout=10): - self.host = host - self.user = user - self.password = password - self.use_system_keys = use_system_keys - self.private_key = private_key - self.use_proxy = use_proxy - self.proxy_host = proxy_host - self.proxy_user = proxy_user - self.proxy_password = proxy_password - self.timeout = timeout - paramiko.util.log_to_file("paramiko.log") - self.logger = rl.Logger("SSHUtils").getLogger() - - def connect(self): - client = paramiko.SSHClient() - if self.use_system_keys: - client.load_system_host_keys() - elif self.private_key: - client.load_host_keys(self.private_key) + +def get_ssh_client(hostname, username, password=None, jumphost=None): + client = None + try: + if jumphost is None: + client = paramiko.SSHClient() else: - client.load_host_keys('/dev/null') + client = JumpHostHopClient() + client.configure_jump_host(jumphost['ip'], + jumphost['username'], + jumphost['password']) + + if client is None: + raise Exception('Could not connect to client') client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect(hostname, + username=username, + password=password) + return client + except Exception, e: + logger.error(e) + return None - t = self.timeout - proxy = None - if self.use_proxy: - proxy_command = 'ssh -o UserKnownHostsFile=/dev/null ' - '-o StrictHostKeyChecking=no %s@%s -W %s:%s' % (self.proxy_user, - self.proxy_host, - self.host, 22) - proxy = paramiko.ProxyCommand(proxy_command) - self.logger.debug("Proxy command: %s" % proxy_command) - while t > 0: - try: - self.logger.debug( - "Trying to stablish ssh connection to %s..." % self.host) - client.connect(self.host, - username=self.user, - password=self.password, - look_for_keys=True, - sock=proxy, - pkey=self.private_key, - timeout=self.timeout) - self.logger.debug("Successfully connected to %s!" % self.host) - return client - except: - time.sleep(1) - t -= 1 - - if t == 0: - return None - - def scp_put(self, local_path, remote_path): - client = self.connect() - if client: - scp = SCPClient(client.get_transport()) - try: - scp.put(local_path, remote_path) - client.close() - return 0 - except Exception, e: - self.logger.error(e) - client.close() - return 1 - else: - self.logger.error("Cannot stablish ssh connection.") - - def scp_get(self, local_path, remote_path): - client = self.connect() - if client: - scp = SCPClient(client.get_transport()) - try: - scp.get(remote_path, local_path) - client.close() - return 0 - except Exception, e: - self.logger.error(e) - client.close() - return 1 - else: - self.logger.error("Cannot stablish ssh connection.") - return 1 - - def run_remote_cmd(self, command): - client = self.connect() - if client: - try: - stdin, stdout, stderr = client.exec_command(command) - out = '' - for line in stdout.readlines(): - out += line - err = stderr.readlines() - client.close() - return out, err - except: - client.close() - return 1 - else: - self.logger.error("Cannot stablish ssh connection.") - return 1 + +def get_file(ssh_conn, src, dest): + try: + sftp = ssh_conn.open_sftp() + sftp.get(src, dest) + return True + except Exception, e: + logger.error("Error [get_file(ssh_conn, '%s', '%s']: %s" % + (src, dest, e)) + return None + + +def put_file(ssh_conn, src, dest): + try: + sftp = ssh_conn.open_sftp() + sftp.put(src, dest) + return True + except Exception, e: + logger.error("Error [put_file(ssh_conn, '%s', '%s']: %s" % + (src, dest, e)) + return None + + +class JumpHostHopClient(paramiko.SSHClient): + ''' + Connect to a remote server using a jumphost hop + ''' + def __init__(self, *args, **kwargs): + self.logger = rl.Logger("JumpHostHopClient").getLogger() + self.jumphost_ssh = None + self.jumphost_transport = None + self.jumphost_channel = None + self.jumphost_ip = None + self.jumphost_ssh_key = None + self.local_ssh_key = os.path.join(os.getcwd(), 'id_rsa') + super(JumpHostHopClient, self).__init__(*args, **kwargs) + + def configure_jump_host(self, jh_ip, jh_user, jh_pass, + jh_ssh_key='/root/.ssh/id_rsa'): + self.jumphost_ip = jh_ip + self.jumphost_ssh_key = jh_ssh_key + self.jumphost_ssh = paramiko.SSHClient() + self.jumphost_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.jumphost_ssh.connect(jh_ip, + username=jh_user, + password=jh_pass) + self.jumphost_transport = self.jumphost_ssh.get_transport() + + def connect(self, hostname, port=22, username='root', password=None, + pkey=None, key_filename=None, timeout=None, allow_agent=True, + look_for_keys=True, compress=False, sock=None, gss_auth=False, + gss_kex=False, gss_deleg_creds=True, gss_host=None, + banner_timeout=None): + try: + if self.jumphost_ssh is None: + raise Exception('You must configure the jump ' + 'host before calling connect') + + get_file_res = get_file(self.jumphost_ssh, + self.jumphost_ssh_key, + self.local_ssh_key) + if get_file_res is None: + raise Exception('Could\'t fetch SSH key from jump host') + jumphost_key = (paramiko.RSAKey + .from_private_key_file(self.local_ssh_key)) + + self.jumphost_channel = self.jumphost_transport.open_channel( + "direct-tcpip", + (hostname, 22), + (self.jumphost_ip, 22)) + + self.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + super(JumpHostHopClient, self).connect(hostname, + username=username, + pkey=jumphost_key, + sock=self.jumphost_channel) + os.remove(self.local_ssh_key) + except Exception, e: + self.logger.error(e) diff --git a/utils/test-sign-artifact.sh b/utils/test-sign-artifact.sh deleted file mode 100755 index f09b7f4e2..000000000 --- a/utils/test-sign-artifact.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -export PATH=$PATH:/usr/local/bin/ - -# clone releng repository -echo "Cloning releng repository..." -[ -d releng ] && rm -rf releng -git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null -#this is where we import the siging key -if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then - source $WORKSPACE/releng/utils/gpg_import_key.sh -fi - -artifact="foo" -echo foo > foo - -testsign () { - echo "Signing artifact: ${artifact}" - gpg2 -vvv --batch \ - --default-key opnfv-helpdesk@rt.linuxfoundation.org \ - --passphrase besteffort \ - --detach-sig $artifact -} - -testsign - |