From d3fc60f22dd0d7831f901568dc9eaa9b748b5215 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Mon, 20 Mar 2017 13:30:55 +0100 Subject: xci: Create skeletons for jobs and rename directory This patch - creates job skeletongs based on the structure of other OPNFV jobs upstream job per scenario triggering and controlling the downstream jobs downstream jobs to run provisioning, deployment and functest - renames infra directory to xci Change-Id: I9a7ac3590d660f4270ab387598586e4270787ecb Signed-off-by: Fatih Degirmenci --- jjb/infra/bifrost-cleanup-job.yml | 140 ------------------------ jjb/infra/bifrost-verify-jobs.yml | 223 -------------------------------------- jjb/infra/bifrost-verify.sh | 134 ----------------------- jjb/xci/bifrost-cleanup-job.yml | 140 ++++++++++++++++++++++++ jjb/xci/bifrost-verify-jobs.yml | 223 ++++++++++++++++++++++++++++++++++++++ jjb/xci/bifrost-verify.sh | 134 +++++++++++++++++++++++ jjb/xci/xci-daily-jobs.yml | 218 +++++++++++++++++++++++++++++++++++++ jjb/xci/xci-deploy.sh | 3 + jjb/xci/xci-functest.sh | 3 + jjb/xci/xci-provision.sh | 3 + 10 files changed, 724 insertions(+), 497 deletions(-) delete mode 100644 jjb/infra/bifrost-cleanup-job.yml delete mode 100644 jjb/infra/bifrost-verify-jobs.yml delete mode 100755 jjb/infra/bifrost-verify.sh create mode 100644 jjb/xci/bifrost-cleanup-job.yml create mode 100644 jjb/xci/bifrost-verify-jobs.yml create mode 100755 jjb/xci/bifrost-verify.sh create mode 100644 jjb/xci/xci-daily-jobs.yml create mode 100755 jjb/xci/xci-deploy.sh create mode 100755 jjb/xci/xci-functest.sh create mode 100755 jjb/xci/xci-provision.sh diff --git a/jjb/infra/bifrost-cleanup-job.yml b/jjb/infra/bifrost-cleanup-job.yml deleted file mode 100644 index 571e275da..000000000 --- a/jjb/infra/bifrost-cleanup-job.yml +++ /dev/null @@ -1,140 +0,0 @@ -- project: - name: 'openstack-bifrost-cleanup' -#-------------------------------- -# branches -#-------------------------------- - stream: - - master: - branch: '{stream}' - -#-------------------------------- -# projects -#-------------------------------- - project: - - 'openstack': - project-repo: 'https://git.openstack.org/openstack/bifrost' - clone-location: '/opt/bifrost' - - 'opnfv': - project-repo: 'https://gerrit.opnfv.org/gerrit/releng' - clone-location: '/opt/releng' - -#-------------------------------- -# jobs -#-------------------------------- - jobs: - - '{project}-bifrost-cleanup-{stream}' - -- job-template: - name: '{project}-bifrost-cleanup-{stream}' - - concurrent: false - - node: bifrost-verify-virtual - - # Make sure no verify job is running on any of the slaves since that would - # produce build logs after we wipe the destination directory. - properties: - - build-blocker: - blocking-jobs: - - '{project}-bifrost-verify-*' - - parameters: - - string: - name: PROJECT - default: '{project}' - - builders: - - shell: | - #!/bin/bash - - set -eu - - # DO NOT change this unless you know what you are doing. - BIFROST_GS_URL="gs://artifacts.opnfv.org/cross-community-ci/openstack/bifrost/$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/" - - # This should never happen... even 'recheck' uses the last jobs' - # gerrit information. Better exit with error so we can investigate - [[ ! -n $GERRIT_NAME ]] || [[ ! -n $GERRIT_CHANGE_NUMBER ]] && exit 1 - - echo "Removing build artifacts for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER" - - if ! [[ "$BIFROST_GS_URL" =~ "/cross-community-ci/openstack/bifrost/" ]]; then - echo "Oops! BIFROST_GS_URL=$BIFROST_GS_URL does not seem like a valid" - echo "bifrost location on the Google storage server. Please double-check" - echo "that it's set properly or fix this line if necessary." - echo "gsutil will not be executed until this is fixed!" - exit 1 - fi - # No force (-f). We always verify upstream jobs so if there are no logs - # something else went wrong and we need to break immediately and investigate - gsutil -m rm -r $BIFROST_GS_URL - - triggers: - - '{project}-gerrit-trigger-cleanup': - branch: '{branch}' - - publishers: - - email: - recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com zhang.jun3g@zte.com.cn -#-------------------------------- -# trigger macros -#-------------------------------- -- trigger: - name: 'openstack-gerrit-trigger-cleanup' - triggers: - - gerrit: - server-name: 'review.openstack.org' - escape-quotes: true - trigger-on: - # We only run this when the change is merged or - # abandoned since we don't need the logs anymore - - patchset-uploaded-event: 'false' - - change-merged-event: 'true' - - change-abandoned-event: 'true' - - change-restored-event: 'false' - - draft-published-event: 'false' - # This is an OPNFV maintenance job. We don't want to provide - # feedback on Gerrit - silent: true - silent-start: true - projects: - - project-compare-type: 'PLAIN' - project-pattern: 'openstack/bifrost' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'true' - forbidden-file-paths: - - compare-type: ANT - pattern: 'doc/**' - - compare-type: ANT - pattern: 'releasenotes/**' - disable-strict-forbidden-file-verification: 'true' - readable-message: true -- trigger: - name: 'opnfv-gerrit-trigger-cleanup' - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - # We only run this when the change is merged or - # abandoned since we don't need the logs anymore - - patchset-uploaded-event: 'false' - - change-merged-event: 'true' - - change-abandoned-event: 'true' - - change-restored-event: 'false' - - draft-published-event: 'false' - # This is an OPNFV maintenance job. We don't want to provide - # feedback on Gerrit - silent: true - silent-start: true - projects: - - project-compare-type: 'ANT' - project-pattern: 'releng' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: 'prototypes/bifrost/**' - readable-message: true diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml deleted file mode 100644 index 33032bc7b..000000000 --- a/jjb/infra/bifrost-verify-jobs.yml +++ /dev/null @@ -1,223 +0,0 @@ -- project: - name: 'openstack-bifrost-verify' -#-------------------------------- -# branches -#-------------------------------- - stream: - - master: - branch: '{stream}' -#-------------------------------- -# projects -#-------------------------------- - project: - - 'openstack': - project-repo: 'https://git.openstack.org/openstack/bifrost' - clone-location: '/opt/bifrost' - - 'opnfv': - project-repo: 'https://gerrit.opnfv.org/gerrit/releng' - clone-location: '/opt/releng' -#-------------------------------- -# distros -#-------------------------------- - distro: - - 'trusty': - disabled: false - dib-os-release: 'trusty' - dib-os-element: 'ubuntu-minimal' - dib-os-packages: 'vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' - extra-dib-elements: 'openssh-server' - - 'centos7': - disabled: false - dib-os-release: '7' - dib-os-element: 'centos7' - dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' - extra-dib-elements: 'openssh-server' - - 'suse': - disabled: false - dib-os-release: '42.2' - dib-os-element: 'opensuse-minimal' - dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' - extra-dib-elements: 'openssh-server' -#-------------------------------- -# type -#-------------------------------- - type: - - virtual -#-------------------------------- -# jobs -#-------------------------------- - jobs: - - '{project}-bifrost-verify-{distro}-{type}-{stream}' - -#-------------------------------- -# VM defaults -#-------------------------------- -- defaults: - name: vm_defaults - vm-disk: '30' - vm-disk-cache: 'unsafe' - vm-memory: '4096' - vm-cpu: '2' - -#-------------------------------- -# job templates -#-------------------------------- -- job-template: - name: '{project}-bifrost-verify-{distro}-{type}-{stream}' - - disabled: '{obj:disabled}' - - defaults: vm_defaults - - concurrent: false - - properties: - - logrotate-default - - build-blocker: - use-build-blocker: true - blocking-jobs: - - '.*-bifrost-verify.*-{type}' - block-level: 'NODE' - - parameters: - - string: - name: PROJECT - default: '{project}' - - string: - name: PROJECT_REPO - default: '{project-repo}' - - string: - name: CLONE_LOCATION - default: '{clone-location}' - - string: - name: DISTRO - default: '{distro}' - - string: - name: DIB_OS_RELEASE - default: '{dib-os-release}' - - string: - name: DIB_OS_ELEMENT - default: '{dib-os-element}' - - string: - name: EXTRA_DIB_ELEMENTS - default: '{extra-dib-elements}' - - string: - name: DIB_OS_PACKAGES - default: '{dib-os-packages}' - - string: - name: VM_DISK - default: '{vm-disk}' - - string: - name: VM_DISK_CACHE - default: '{vm-disk-cache}' - - string: - name: VM_MEMORY - default: '{vm-memory}' - - string: - name: VM_CPU - default: '{vm-cpu}' - - string: - name: CLEAN_DIB_IMAGES - default: 'true' - - label: - name: SLAVE_LABEL - default: 'infra-{type}-{distro}' - - string: - name: BIFROST_LOG_URL - default: 'http://artifacts.opnfv.org/cross-community-ci/openstack/bifrost/$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER/$JOB_NAME' - - scm: - - git: - url: '$PROJECT_REPO' - refspec: '$GERRIT_REFSPEC' - branches: - - 'origin/$BRANCH' - skip-tag: true - choosing-strategy: 'gerrit' - timeout: 10 - wipe-workspace: true - - triggers: - - '{project}-gerrit-trigger': - branch: '{branch}' - - builders: - - bifrost-set-name - - bifrost-build - - publishers: - - email: - recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com -#-------------------------------- -# trigger macros -#-------------------------------- -- trigger: - name: 'openstack-gerrit-trigger' - triggers: - - gerrit: - server-name: 'review.openstack.org' - escape-quotes: true - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - comment-added-contains-event: - comment-contains-value: 'recheck' - custom-url: '* $JOB_NAME $BIFROST_LOG_URL/index.html' - silent-start: true - projects: - - project-compare-type: 'PLAIN' - project-pattern: 'openstack/bifrost' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'true' - forbidden-file-paths: - - compare-type: ANT - pattern: 'doc/**' - - compare-type: ANT - pattern: 'releasenotes/**' - disable-strict-forbidden-file-verification: 'true' - readable-message: true -- trigger: - name: 'opnfv-gerrit-trigger' - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - custom-url: '* $JOB_NAME $BIFROST_LOG_URL/index.html' - projects: - - project-compare-type: 'ANT' - project-pattern: 'releng' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: 'prototypes/bifrost/**' - readable-message: true - -#--------------------------- -# builder macros -#--------------------------- -- builder: - name: bifrost-set-name - builders: - - description-setter: - description: "Built on $NODE_NAME" - -- builder: - name: bifrost-build - builders: - - shell: - !include-raw: ./bifrost-verify.sh diff --git a/jjb/infra/bifrost-verify.sh b/jjb/infra/bifrost-verify.sh deleted file mode 100755 index 4115ffcc4..000000000 --- a/jjb/infra/bifrost-verify.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# SPDX-license-identifier: Apache-2.0 -############################################################################## -# Copyright (c) 2016 Ericsson AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -o errexit -set -o nounset -set -o pipefail - -trap cleanup_and_upload EXIT - -function upload_logs() { - BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText" - BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:} - - # Make sure the old landing page is gone in case - # we break later on. We don't want to publish - # stale information. - # TODO: Maybe cleanup the entire $BIFROST_GS_URL directory - # before we upload the new data. - gsutil -q rm ${BIFROST_GS_URL}/index.html || true - - echo "Uploading collected bifrost build logs to ${BIFROST_LOG_URL}" - - if [[ -d ${WORKSPACE}/logs ]]; then - pushd ${WORKSPACE}/logs &> /dev/null - for x in *.log; do - echo "Compressing and uploading $x" - gsutil -q cp -Z ${x} ${BIFROST_GS_URL}/${x} - done - popd &> /dev/null - fi - - echo "Generating the ${BIFROST_LOG_URL}/index.html landing page" - cat > ${WORKSPACE}/index.html < -

Build results for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER

-

Job: $JOB_NAME

-
    -
  • build_log.txt
    • -EOF - - if [[ -d ${WORKSPACE}/logs ]]; then - pushd ${WORKSPACE}/logs &> /dev/null - for x in *.log; do - echo "
  • ${x}
    • " >> ${WORKSPACE}/index.html - done - popd &> /dev/null - fi - - cat >> ${WORKSPACE}/index.html << EOF -
- -EOF - - # Finally, download and upload the entire build log so we can retain - # as much build information as possible - echo "Uploading the final console output" - curl -s -L ${BIFROST_CONSOLE_LOG} > ${WORKSPACE}/build_log.txt - gsutil -q cp -Z ${WORKSPACE}/build_log.txt ${BIFROST_GS_URL}/build_log.txt - rm ${WORKSPACE}/build_log.txt - - # Upload landing page - gsutil -q cp ${WORKSPACE}/index.html ${BIFROST_GS_URL}/index.html - rm ${WORKSPACE}/index.html -} - -function fix_ownership() { - if [ -z "${JOB_URL+x}" ]; then - echo "Not running as part of Jenkins. Handle the logs manually." - else - # Make sure cache exists - [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache - - sudo chown -R jenkins:jenkins $WORKSPACE - sudo chown -R jenkins:jenkins ${HOME}/.cache - fi -} - -function cleanup_and_upload() { - original_exit=$? - fix_ownership - upload_logs - exit $original_exit -} - -# check distro to see if we support it -if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then - echo "Distro $DISTRO is not supported!" - exit 1 -fi - -# remove previously cloned repos -sudo /bin/rm -rf /opt/bifrost /opt/puppet-infracloud /opt/stack /opt/releng - -# Fix up permissions -fix_ownership - -# clone all the repos first and checkout the patch afterwards -sudo git clone https://git.openstack.org/openstack/bifrost /opt/bifrost -sudo git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud -sudo git clone https://gerrit.opnfv.org/gerrit/releng /opt/releng - -# checkout the patch -cd $CLONE_LOCATION -sudo git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD - -# combine opnfv and upstream scripts/playbooks -sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/ - -# place bridge creation file on the right path -sudo mkdir -p /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt -sudo cp /opt/puppet-infracloud/templates/bifrost/create_bridge.py.erb /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py - -# replace bridge name -sudo sed -i s/"<%= @bridge_name -%>"/br_opnfv/g /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py - -# cleanup remnants of previous deployment -cd /opt/bifrost -sudo -E ./scripts/destroy-env.sh - -# provision 3 VMs; jumphost, controller, and compute -cd /opt/bifrost -sudo -E ./scripts/test-bifrost-deployment.sh - -# list the provisioned VMs -cd /opt/bifrost -source env-vars -ironic node-list -virsh list diff --git a/jjb/xci/bifrost-cleanup-job.yml b/jjb/xci/bifrost-cleanup-job.yml new file mode 100644 index 000000000..571e275da --- /dev/null +++ b/jjb/xci/bifrost-cleanup-job.yml @@ -0,0 +1,140 @@ +- project: + name: 'openstack-bifrost-cleanup' +#-------------------------------- +# branches +#-------------------------------- + stream: + - master: + branch: '{stream}' + +#-------------------------------- +# projects +#-------------------------------- + project: + - 'openstack': + project-repo: 'https://git.openstack.org/openstack/bifrost' + clone-location: '/opt/bifrost' + - 'opnfv': + project-repo: 'https://gerrit.opnfv.org/gerrit/releng' + clone-location: '/opt/releng' + +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - '{project}-bifrost-cleanup-{stream}' + +- job-template: + name: '{project}-bifrost-cleanup-{stream}' + + concurrent: false + + node: bifrost-verify-virtual + + # Make sure no verify job is running on any of the slaves since that would + # produce build logs after we wipe the destination directory. + properties: + - build-blocker: + blocking-jobs: + - '{project}-bifrost-verify-*' + + parameters: + - string: + name: PROJECT + default: '{project}' + + builders: + - shell: | + #!/bin/bash + + set -eu + + # DO NOT change this unless you know what you are doing. + BIFROST_GS_URL="gs://artifacts.opnfv.org/cross-community-ci/openstack/bifrost/$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/" + + # This should never happen... even 'recheck' uses the last jobs' + # gerrit information. Better exit with error so we can investigate + [[ ! -n $GERRIT_NAME ]] || [[ ! -n $GERRIT_CHANGE_NUMBER ]] && exit 1 + + echo "Removing build artifacts for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER" + + if ! [[ "$BIFROST_GS_URL" =~ "/cross-community-ci/openstack/bifrost/" ]]; then + echo "Oops! BIFROST_GS_URL=$BIFROST_GS_URL does not seem like a valid" + echo "bifrost location on the Google storage server. Please double-check" + echo "that it's set properly or fix this line if necessary." + echo "gsutil will not be executed until this is fixed!" + exit 1 + fi + # No force (-f). We always verify upstream jobs so if there are no logs + # something else went wrong and we need to break immediately and investigate + gsutil -m rm -r $BIFROST_GS_URL + + triggers: + - '{project}-gerrit-trigger-cleanup': + branch: '{branch}' + + publishers: + - email: + recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com zhang.jun3g@zte.com.cn +#-------------------------------- +# trigger macros +#-------------------------------- +- trigger: + name: 'openstack-gerrit-trigger-cleanup' + triggers: + - gerrit: + server-name: 'review.openstack.org' + escape-quotes: true + trigger-on: + # We only run this when the change is merged or + # abandoned since we don't need the logs anymore + - patchset-uploaded-event: 'false' + - change-merged-event: 'true' + - change-abandoned-event: 'true' + - change-restored-event: 'false' + - draft-published-event: 'false' + # This is an OPNFV maintenance job. We don't want to provide + # feedback on Gerrit + silent: true + silent-start: true + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'openstack/bifrost' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'doc/**' + - compare-type: ANT + pattern: 'releasenotes/**' + disable-strict-forbidden-file-verification: 'true' + readable-message: true +- trigger: + name: 'opnfv-gerrit-trigger-cleanup' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + # We only run this when the change is merged or + # abandoned since we don't need the logs anymore + - patchset-uploaded-event: 'false' + - change-merged-event: 'true' + - change-abandoned-event: 'true' + - change-restored-event: 'false' + - draft-published-event: 'false' + # This is an OPNFV maintenance job. We don't want to provide + # feedback on Gerrit + silent: true + silent-start: true + projects: + - project-compare-type: 'ANT' + project-pattern: 'releng' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'prototypes/bifrost/**' + readable-message: true diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml new file mode 100644 index 000000000..33032bc7b --- /dev/null +++ b/jjb/xci/bifrost-verify-jobs.yml @@ -0,0 +1,223 @@ +- project: + name: 'openstack-bifrost-verify' +#-------------------------------- +# branches +#-------------------------------- + stream: + - master: + branch: '{stream}' +#-------------------------------- +# projects +#-------------------------------- + project: + - 'openstack': + project-repo: 'https://git.openstack.org/openstack/bifrost' + clone-location: '/opt/bifrost' + - 'opnfv': + project-repo: 'https://gerrit.opnfv.org/gerrit/releng' + clone-location: '/opt/releng' +#-------------------------------- +# distros +#-------------------------------- + distro: + - 'trusty': + disabled: false + dib-os-release: 'trusty' + dib-os-element: 'ubuntu-minimal' + dib-os-packages: 'vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'centos7': + disabled: false + dib-os-release: '7' + dib-os-element: 'centos7' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'suse': + disabled: false + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' +#-------------------------------- +# type +#-------------------------------- + type: + - virtual +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - '{project}-bifrost-verify-{distro}-{type}-{stream}' + +#-------------------------------- +# VM defaults +#-------------------------------- +- defaults: + name: vm_defaults + vm-disk: '30' + vm-disk-cache: 'unsafe' + vm-memory: '4096' + vm-cpu: '2' + +#-------------------------------- +# job templates +#-------------------------------- +- job-template: + name: '{project}-bifrost-verify-{distro}-{type}-{stream}' + + disabled: '{obj:disabled}' + + defaults: vm_defaults + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '.*-bifrost-verify.*-{type}' + block-level: 'NODE' + + parameters: + - string: + name: PROJECT + default: '{project}' + - string: + name: PROJECT_REPO + default: '{project-repo}' + - string: + name: CLONE_LOCATION + default: '{clone-location}' + - string: + name: DISTRO + default: '{distro}' + - string: + name: DIB_OS_RELEASE + default: '{dib-os-release}' + - string: + name: DIB_OS_ELEMENT + default: '{dib-os-element}' + - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: + name: DIB_OS_PACKAGES + default: '{dib-os-packages}' + - string: + name: VM_DISK + default: '{vm-disk}' + - string: + name: VM_DISK_CACHE + default: '{vm-disk-cache}' + - string: + name: VM_MEMORY + default: '{vm-memory}' + - string: + name: VM_CPU + default: '{vm-cpu}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - label: + name: SLAVE_LABEL + default: 'infra-{type}-{distro}' + - string: + name: BIFROST_LOG_URL + default: 'http://artifacts.opnfv.org/cross-community-ci/openstack/bifrost/$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER/$JOB_NAME' + + scm: + - git: + url: '$PROJECT_REPO' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$BRANCH' + skip-tag: true + choosing-strategy: 'gerrit' + timeout: 10 + wipe-workspace: true + + triggers: + - '{project}-gerrit-trigger': + branch: '{branch}' + + builders: + - bifrost-set-name + - bifrost-build + + publishers: + - email: + recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com +#-------------------------------- +# trigger macros +#-------------------------------- +- trigger: + name: 'openstack-gerrit-trigger' + triggers: + - gerrit: + server-name: 'review.openstack.org' + escape-quotes: true + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - comment-added-contains-event: + comment-contains-value: 'recheck' + custom-url: '* $JOB_NAME $BIFROST_LOG_URL/index.html' + silent-start: true + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'openstack/bifrost' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'doc/**' + - compare-type: ANT + pattern: 'releasenotes/**' + disable-strict-forbidden-file-verification: 'true' + readable-message: true +- trigger: + name: 'opnfv-gerrit-trigger' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + custom-url: '* $JOB_NAME $BIFROST_LOG_URL/index.html' + projects: + - project-compare-type: 'ANT' + project-pattern: 'releng' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'prototypes/bifrost/**' + readable-message: true + +#--------------------------- +# builder macros +#--------------------------- +- builder: + name: bifrost-set-name + builders: + - description-setter: + description: "Built on $NODE_NAME" + +- builder: + name: bifrost-build + builders: + - shell: + !include-raw: ./bifrost-verify.sh diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh new file mode 100755 index 000000000..4115ffcc4 --- /dev/null +++ b/jjb/xci/bifrost-verify.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +trap cleanup_and_upload EXIT + +function upload_logs() { + BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText" + BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:} + + # Make sure the old landing page is gone in case + # we break later on. We don't want to publish + # stale information. + # TODO: Maybe cleanup the entire $BIFROST_GS_URL directory + # before we upload the new data. + gsutil -q rm ${BIFROST_GS_URL}/index.html || true + + echo "Uploading collected bifrost build logs to ${BIFROST_LOG_URL}" + + if [[ -d ${WORKSPACE}/logs ]]; then + pushd ${WORKSPACE}/logs &> /dev/null + for x in *.log; do + echo "Compressing and uploading $x" + gsutil -q cp -Z ${x} ${BIFROST_GS_URL}/${x} + done + popd &> /dev/null + fi + + echo "Generating the ${BIFROST_LOG_URL}/index.html landing page" + cat > ${WORKSPACE}/index.html < +

Build results for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER

+

Job: $JOB_NAME

+
    +
  • build_log.txt
    • +EOF + + if [[ -d ${WORKSPACE}/logs ]]; then + pushd ${WORKSPACE}/logs &> /dev/null + for x in *.log; do + echo "
  • ${x}
    • " >> ${WORKSPACE}/index.html + done + popd &> /dev/null + fi + + cat >> ${WORKSPACE}/index.html << EOF +
+ +EOF + + # Finally, download and upload the entire build log so we can retain + # as much build information as possible + echo "Uploading the final console output" + curl -s -L ${BIFROST_CONSOLE_LOG} > ${WORKSPACE}/build_log.txt + gsutil -q cp -Z ${WORKSPACE}/build_log.txt ${BIFROST_GS_URL}/build_log.txt + rm ${WORKSPACE}/build_log.txt + + # Upload landing page + gsutil -q cp ${WORKSPACE}/index.html ${BIFROST_GS_URL}/index.html + rm ${WORKSPACE}/index.html +} + +function fix_ownership() { + if [ -z "${JOB_URL+x}" ]; then + echo "Not running as part of Jenkins. Handle the logs manually." + else + # Make sure cache exists + [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache + + sudo chown -R jenkins:jenkins $WORKSPACE + sudo chown -R jenkins:jenkins ${HOME}/.cache + fi +} + +function cleanup_and_upload() { + original_exit=$? + fix_ownership + upload_logs + exit $original_exit +} + +# check distro to see if we support it +if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then + echo "Distro $DISTRO is not supported!" + exit 1 +fi + +# remove previously cloned repos +sudo /bin/rm -rf /opt/bifrost /opt/puppet-infracloud /opt/stack /opt/releng + +# Fix up permissions +fix_ownership + +# clone all the repos first and checkout the patch afterwards +sudo git clone https://git.openstack.org/openstack/bifrost /opt/bifrost +sudo git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud +sudo git clone https://gerrit.opnfv.org/gerrit/releng /opt/releng + +# checkout the patch +cd $CLONE_LOCATION +sudo git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD + +# combine opnfv and upstream scripts/playbooks +sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/ + +# place bridge creation file on the right path +sudo mkdir -p /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt +sudo cp /opt/puppet-infracloud/templates/bifrost/create_bridge.py.erb /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py + +# replace bridge name +sudo sed -i s/"<%= @bridge_name -%>"/br_opnfv/g /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py + +# cleanup remnants of previous deployment +cd /opt/bifrost +sudo -E ./scripts/destroy-env.sh + +# provision 3 VMs; jumphost, controller, and compute +cd /opt/bifrost +sudo -E ./scripts/test-bifrost-deployment.sh + +# list the provisioned VMs +cd /opt/bifrost +source env-vars +ironic node-list +virsh list diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml new file mode 100644 index 000000000..92fa80e74 --- /dev/null +++ b/jjb/xci/xci-daily-jobs.yml @@ -0,0 +1,218 @@ +- project: + name: 'bifrost-osa-daily' +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + ocata: &ocata + stream: ocata + branch: 'stable/{stream}' + gs-pathname: '/{stream}' +#-------------------------------- +# scenarios +#-------------------------------- + scenario: + # HA scenarios + - 'os-nosdn-nofeature-ha': + auto-trigger-name: 'daily-trigger-disabled' +#-------------------------------- +# XCI PODs +#-------------------------------- + pod: + - virtual: + <<: *master +#-------------------------------- +# Supported Distros +#-------------------------------- + distro: + - 'xenial': + disabled: false + slave-label: xci-xenial-virtual + dib-os-release: 'xenial' + dib-os-element: 'ubuntu-minimal' + dib-os-packages: 'vlan,vim,less,bridge-utils,sudo,language-pack-en,iputils-ping,rsyslog,curl,python,debootstrap,ifenslave,ifenslave-2.6,lsof,lvm2,tcpdump,nfs-kernel-server,chrony' + extra-dib-elements: 'openssh-server' + - 'centos7': + disabled: true + slave-label: xci-centos7-virtual + dib-os-release: '7' + dib-os-element: 'centos7' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'suse': + disabled: true + slave-label: xci-suse-virtual + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' +#-------------------------------- +# Phases +#-------------------------------- + phase: + - 'provision': + project: 'openstack' + project-repo: 'https://git.openstack.org/openstack/bifrost' + project-branch: '{branch}' + clone-location: '/opt/bifrost' + - 'deploy': + project: 'openstack' + project-repo: 'https://git.openstack.org/openstack/openstack-ansible' + project-branch: '{branch}' + clone-location: '/opt/openstack-ansible' + - 'functest': + project: 'opnfv' + project-repo: 'https://gerrit.opnfv.org/gerrit/functest' + project-branch: 'master' + clone-location: '/opt/functest' +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - 'xci-{scenario}-{pod}-{distro}-daily-{stream}' + - 'xci-{phase}-{pod}-{distro}-daily-{stream}' + +#-------------------------------- +# job templates +#-------------------------------- +- job-template: + name: 'xci-{scenario}-{pod}-{distro}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-os-.*?-{pod}-daily-.*' + block-level: 'NODE' + + parameters: + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - label: + name: SLAVE_LABEL + default: '{slave-label}' + + triggers: + - '{auto-trigger-name}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - trigger-builds: + - project: 'xci-provision-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + - trigger-builds: + - project: 'xci-deploy-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + - trigger-builds: + - project: 'xci-functest-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + + publishers: + - email: + recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com + +- job-template: + name: 'xci-{phase}-{pod}-{distro}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-provision-.*?-{pod}-daily-.*' + - 'xci-deploy-.*?-{pod}-daily-.*' + - 'xci-functest-.*?-{pod}-daily-.*' + block-level: 'NODE' + + parameters: + - string: + name: PROJECT + default: '{project}' + - string: + name: PROJECT_REPO + default: '{project-repo}' + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - string: + name: CLONE_LOCATION + default: '{clone-location}' + - string: + name: DISTRO + default: '{distro}' + - string: + name: DIB_OS_RELEASE + default: '{dib-os-release}' + - string: + name: DIB_OS_ELEMENT + default: '{dib-os-element}' + - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: + name: DIB_OS_PACKAGES + default: '{dib-os-packages}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + + scm: + - git: + url: '$PROJECT_REPO' + branches: + - 'origin/{project-branch}' + wipe-workspace: true + timeout: 15 + + builders: + - description-setter: + description: "Built on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" + - 'xci-{phase}-builder' +#--------------------------- +# builder macros +#--------------------------- +- builder: + name: xci-provision-builder + builders: + - shell: + !include-raw: ./xci-provision.sh +- builder: + name: xci-deploy-builder + builders: + - shell: + !include-raw: ./xci-deploy.sh +- builder: + name: xci-functest-builder + builders: + - shell: + !include-raw: ./xci-functest.sh diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh new file mode 100755 index 000000000..b97b24e47 --- /dev/null +++ b/jjb/xci/xci-deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Deployment via openstack-ansible" diff --git a/jjb/xci/xci-functest.sh b/jjb/xci/xci-functest.sh new file mode 100755 index 000000000..0f58dfefc --- /dev/null +++ b/jjb/xci/xci-functest.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Functional testing with functest" diff --git a/jjb/xci/xci-provision.sh b/jjb/xci/xci-provision.sh new file mode 100755 index 000000000..b77c79caa --- /dev/null +++ b/jjb/xci/xci-provision.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Provisioning via bifrost" -- cgit 1.2.3-korg