From d3fc60f22dd0d7831f901568dc9eaa9b748b5215 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Mon, 20 Mar 2017 13:30:55 +0100 Subject: xci: Create skeletons for jobs and rename directory This patch - creates job skeletongs based on the structure of other OPNFV jobs upstream job per scenario triggering and controlling the downstream jobs downstream jobs to run provisioning, deployment and functest - renames infra directory to xci Change-Id: I9a7ac3590d660f4270ab387598586e4270787ecb Signed-off-by: Fatih Degirmenci --- jjb/xci/bifrost-cleanup-job.yml | 140 +++++++++++++++++++++++++ jjb/xci/bifrost-verify-jobs.yml | 223 ++++++++++++++++++++++++++++++++++++++++ jjb/xci/bifrost-verify.sh | 134 ++++++++++++++++++++++++ jjb/xci/xci-daily-jobs.yml | 218 +++++++++++++++++++++++++++++++++++++++ jjb/xci/xci-deploy.sh | 3 + jjb/xci/xci-functest.sh | 3 + jjb/xci/xci-provision.sh | 3 + 7 files changed, 724 insertions(+) create mode 100644 jjb/xci/bifrost-cleanup-job.yml create mode 100644 jjb/xci/bifrost-verify-jobs.yml create mode 100755 jjb/xci/bifrost-verify.sh create mode 100644 jjb/xci/xci-daily-jobs.yml create mode 100755 jjb/xci/xci-deploy.sh create mode 100755 jjb/xci/xci-functest.sh create mode 100755 jjb/xci/xci-provision.sh (limited to 'jjb/xci') diff --git a/jjb/xci/bifrost-cleanup-job.yml b/jjb/xci/bifrost-cleanup-job.yml new file mode 100644 index 000000000..571e275da --- /dev/null +++ b/jjb/xci/bifrost-cleanup-job.yml @@ -0,0 +1,140 @@ +- project: + name: 'openstack-bifrost-cleanup' +#-------------------------------- +# branches +#-------------------------------- + stream: + - master: + branch: '{stream}' + +#-------------------------------- +# projects +#-------------------------------- + project: + - 'openstack': + project-repo: 'https://git.openstack.org/openstack/bifrost' + clone-location: '/opt/bifrost' + - 'opnfv': + project-repo: 'https://gerrit.opnfv.org/gerrit/releng' + clone-location: '/opt/releng' + +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - '{project}-bifrost-cleanup-{stream}' + +- job-template: + name: '{project}-bifrost-cleanup-{stream}' + + concurrent: false + + node: bifrost-verify-virtual + + # Make sure no verify job is running on any of the slaves since that would + # produce build logs after we wipe the destination directory. + properties: + - build-blocker: + blocking-jobs: + - '{project}-bifrost-verify-*' + + parameters: + - string: + name: PROJECT + default: '{project}' + + builders: + - shell: | + #!/bin/bash + + set -eu + + # DO NOT change this unless you know what you are doing. + BIFROST_GS_URL="gs://artifacts.opnfv.org/cross-community-ci/openstack/bifrost/$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/" + + # This should never happen... even 'recheck' uses the last jobs' + # gerrit information. Better exit with error so we can investigate + [[ ! -n $GERRIT_NAME ]] || [[ ! -n $GERRIT_CHANGE_NUMBER ]] && exit 1 + + echo "Removing build artifacts for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER" + + if ! [[ "$BIFROST_GS_URL" =~ "/cross-community-ci/openstack/bifrost/" ]]; then + echo "Oops! BIFROST_GS_URL=$BIFROST_GS_URL does not seem like a valid" + echo "bifrost location on the Google storage server. Please double-check" + echo "that it's set properly or fix this line if necessary." + echo "gsutil will not be executed until this is fixed!" + exit 1 + fi + # No force (-f). We always verify upstream jobs so if there are no logs + # something else went wrong and we need to break immediately and investigate + gsutil -m rm -r $BIFROST_GS_URL + + triggers: + - '{project}-gerrit-trigger-cleanup': + branch: '{branch}' + + publishers: + - email: + recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com zhang.jun3g@zte.com.cn +#-------------------------------- +# trigger macros +#-------------------------------- +- trigger: + name: 'openstack-gerrit-trigger-cleanup' + triggers: + - gerrit: + server-name: 'review.openstack.org' + escape-quotes: true + trigger-on: + # We only run this when the change is merged or + # abandoned since we don't need the logs anymore + - patchset-uploaded-event: 'false' + - change-merged-event: 'true' + - change-abandoned-event: 'true' + - change-restored-event: 'false' + - draft-published-event: 'false' + # This is an OPNFV maintenance job. We don't want to provide + # feedback on Gerrit + silent: true + silent-start: true + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'openstack/bifrost' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'doc/**' + - compare-type: ANT + pattern: 'releasenotes/**' + disable-strict-forbidden-file-verification: 'true' + readable-message: true +- trigger: + name: 'opnfv-gerrit-trigger-cleanup' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + # We only run this when the change is merged or + # abandoned since we don't need the logs anymore + - patchset-uploaded-event: 'false' + - change-merged-event: 'true' + - change-abandoned-event: 'true' + - change-restored-event: 'false' + - draft-published-event: 'false' + # This is an OPNFV maintenance job. We don't want to provide + # feedback on Gerrit + silent: true + silent-start: true + projects: + - project-compare-type: 'ANT' + project-pattern: 'releng' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'prototypes/bifrost/**' + readable-message: true diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml new file mode 100644 index 000000000..33032bc7b --- /dev/null +++ b/jjb/xci/bifrost-verify-jobs.yml @@ -0,0 +1,223 @@ +- project: + name: 'openstack-bifrost-verify' +#-------------------------------- +# branches +#-------------------------------- + stream: + - master: + branch: '{stream}' +#-------------------------------- +# projects +#-------------------------------- + project: + - 'openstack': + project-repo: 'https://git.openstack.org/openstack/bifrost' + clone-location: '/opt/bifrost' + - 'opnfv': + project-repo: 'https://gerrit.opnfv.org/gerrit/releng' + clone-location: '/opt/releng' +#-------------------------------- +# distros +#-------------------------------- + distro: + - 'trusty': + disabled: false + dib-os-release: 'trusty' + dib-os-element: 'ubuntu-minimal' + dib-os-packages: 'vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'centos7': + disabled: false + dib-os-release: '7' + dib-os-element: 'centos7' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'suse': + disabled: false + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' +#-------------------------------- +# type +#-------------------------------- + type: + - virtual +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - '{project}-bifrost-verify-{distro}-{type}-{stream}' + +#-------------------------------- +# VM defaults +#-------------------------------- +- defaults: + name: vm_defaults + vm-disk: '30' + vm-disk-cache: 'unsafe' + vm-memory: '4096' + vm-cpu: '2' + +#-------------------------------- +# job templates +#-------------------------------- +- job-template: + name: '{project}-bifrost-verify-{distro}-{type}-{stream}' + + disabled: '{obj:disabled}' + + defaults: vm_defaults + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '.*-bifrost-verify.*-{type}' + block-level: 'NODE' + + parameters: + - string: + name: PROJECT + default: '{project}' + - string: + name: PROJECT_REPO + default: '{project-repo}' + - string: + name: CLONE_LOCATION + default: '{clone-location}' + - string: + name: DISTRO + default: '{distro}' + - string: + name: DIB_OS_RELEASE + default: '{dib-os-release}' + - string: + name: DIB_OS_ELEMENT + default: '{dib-os-element}' + - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: + name: DIB_OS_PACKAGES + default: '{dib-os-packages}' + - string: + name: VM_DISK + default: '{vm-disk}' + - string: + name: VM_DISK_CACHE + default: '{vm-disk-cache}' + - string: + name: VM_MEMORY + default: '{vm-memory}' + - string: + name: VM_CPU + default: '{vm-cpu}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - label: + name: SLAVE_LABEL + default: 'infra-{type}-{distro}' + - string: + name: BIFROST_LOG_URL + default: 'http://artifacts.opnfv.org/cross-community-ci/openstack/bifrost/$GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER/$JOB_NAME' + + scm: + - git: + url: '$PROJECT_REPO' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$BRANCH' + skip-tag: true + choosing-strategy: 'gerrit' + timeout: 10 + wipe-workspace: true + + triggers: + - '{project}-gerrit-trigger': + branch: '{branch}' + + builders: + - bifrost-set-name + - bifrost-build + + publishers: + - email: + recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com +#-------------------------------- +# trigger macros +#-------------------------------- +- trigger: + name: 'openstack-gerrit-trigger' + triggers: + - gerrit: + server-name: 'review.openstack.org' + escape-quotes: true + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - comment-added-contains-event: + comment-contains-value: 'recheck' + custom-url: '* $JOB_NAME $BIFROST_LOG_URL/index.html' + silent-start: true + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'openstack/bifrost' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'doc/**' + - compare-type: ANT + pattern: 'releasenotes/**' + disable-strict-forbidden-file-verification: 'true' + readable-message: true +- trigger: + name: 'opnfv-gerrit-trigger' + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + custom-url: '* $JOB_NAME $BIFROST_LOG_URL/index.html' + projects: + - project-compare-type: 'ANT' + project-pattern: 'releng' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'prototypes/bifrost/**' + readable-message: true + +#--------------------------- +# builder macros +#--------------------------- +- builder: + name: bifrost-set-name + builders: + - description-setter: + description: "Built on $NODE_NAME" + +- builder: + name: bifrost-build + builders: + - shell: + !include-raw: ./bifrost-verify.sh diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh new file mode 100755 index 000000000..4115ffcc4 --- /dev/null +++ b/jjb/xci/bifrost-verify.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +trap cleanup_and_upload EXIT + +function upload_logs() { + BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText" + BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:} + + # Make sure the old landing page is gone in case + # we break later on. We don't want to publish + # stale information. + # TODO: Maybe cleanup the entire $BIFROST_GS_URL directory + # before we upload the new data. + gsutil -q rm ${BIFROST_GS_URL}/index.html || true + + echo "Uploading collected bifrost build logs to ${BIFROST_LOG_URL}" + + if [[ -d ${WORKSPACE}/logs ]]; then + pushd ${WORKSPACE}/logs &> /dev/null + for x in *.log; do + echo "Compressing and uploading $x" + gsutil -q cp -Z ${x} ${BIFROST_GS_URL}/${x} + done + popd &> /dev/null + fi + + echo "Generating the ${BIFROST_LOG_URL}/index.html landing page" + cat > ${WORKSPACE}/index.html < +

Build results for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER

+

Job: $JOB_NAME

+
    +
  • build_log.txt
    • +EOF + + if [[ -d ${WORKSPACE}/logs ]]; then + pushd ${WORKSPACE}/logs &> /dev/null + for x in *.log; do + echo "
  • ${x}
    • " >> ${WORKSPACE}/index.html + done + popd &> /dev/null + fi + + cat >> ${WORKSPACE}/index.html << EOF +
+ +EOF + + # Finally, download and upload the entire build log so we can retain + # as much build information as possible + echo "Uploading the final console output" + curl -s -L ${BIFROST_CONSOLE_LOG} > ${WORKSPACE}/build_log.txt + gsutil -q cp -Z ${WORKSPACE}/build_log.txt ${BIFROST_GS_URL}/build_log.txt + rm ${WORKSPACE}/build_log.txt + + # Upload landing page + gsutil -q cp ${WORKSPACE}/index.html ${BIFROST_GS_URL}/index.html + rm ${WORKSPACE}/index.html +} + +function fix_ownership() { + if [ -z "${JOB_URL+x}" ]; then + echo "Not running as part of Jenkins. Handle the logs manually." + else + # Make sure cache exists + [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache + + sudo chown -R jenkins:jenkins $WORKSPACE + sudo chown -R jenkins:jenkins ${HOME}/.cache + fi +} + +function cleanup_and_upload() { + original_exit=$? + fix_ownership + upload_logs + exit $original_exit +} + +# check distro to see if we support it +if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then + echo "Distro $DISTRO is not supported!" + exit 1 +fi + +# remove previously cloned repos +sudo /bin/rm -rf /opt/bifrost /opt/puppet-infracloud /opt/stack /opt/releng + +# Fix up permissions +fix_ownership + +# clone all the repos first and checkout the patch afterwards +sudo git clone https://git.openstack.org/openstack/bifrost /opt/bifrost +sudo git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud +sudo git clone https://gerrit.opnfv.org/gerrit/releng /opt/releng + +# checkout the patch +cd $CLONE_LOCATION +sudo git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD + +# combine opnfv and upstream scripts/playbooks +sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/ + +# place bridge creation file on the right path +sudo mkdir -p /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt +sudo cp /opt/puppet-infracloud/templates/bifrost/create_bridge.py.erb /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py + +# replace bridge name +sudo sed -i s/"<%= @bridge_name -%>"/br_opnfv/g /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py + +# cleanup remnants of previous deployment +cd /opt/bifrost +sudo -E ./scripts/destroy-env.sh + +# provision 3 VMs; jumphost, controller, and compute +cd /opt/bifrost +sudo -E ./scripts/test-bifrost-deployment.sh + +# list the provisioned VMs +cd /opt/bifrost +source env-vars +ironic node-list +virsh list diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml new file mode 100644 index 000000000..92fa80e74 --- /dev/null +++ b/jjb/xci/xci-daily-jobs.yml @@ -0,0 +1,218 @@ +- project: + name: 'bifrost-osa-daily' +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + ocata: &ocata + stream: ocata + branch: 'stable/{stream}' + gs-pathname: '/{stream}' +#-------------------------------- +# scenarios +#-------------------------------- + scenario: + # HA scenarios + - 'os-nosdn-nofeature-ha': + auto-trigger-name: 'daily-trigger-disabled' +#-------------------------------- +# XCI PODs +#-------------------------------- + pod: + - virtual: + <<: *master +#-------------------------------- +# Supported Distros +#-------------------------------- + distro: + - 'xenial': + disabled: false + slave-label: xci-xenial-virtual + dib-os-release: 'xenial' + dib-os-element: 'ubuntu-minimal' + dib-os-packages: 'vlan,vim,less,bridge-utils,sudo,language-pack-en,iputils-ping,rsyslog,curl,python,debootstrap,ifenslave,ifenslave-2.6,lsof,lvm2,tcpdump,nfs-kernel-server,chrony' + extra-dib-elements: 'openssh-server' + - 'centos7': + disabled: true + slave-label: xci-centos7-virtual + dib-os-release: '7' + dib-os-element: 'centos7' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'suse': + disabled: true + slave-label: xci-suse-virtual + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' +#-------------------------------- +# Phases +#-------------------------------- + phase: + - 'provision': + project: 'openstack' + project-repo: 'https://git.openstack.org/openstack/bifrost' + project-branch: '{branch}' + clone-location: '/opt/bifrost' + - 'deploy': + project: 'openstack' + project-repo: 'https://git.openstack.org/openstack/openstack-ansible' + project-branch: '{branch}' + clone-location: '/opt/openstack-ansible' + - 'functest': + project: 'opnfv' + project-repo: 'https://gerrit.opnfv.org/gerrit/functest' + project-branch: 'master' + clone-location: '/opt/functest' +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - 'xci-{scenario}-{pod}-{distro}-daily-{stream}' + - 'xci-{phase}-{pod}-{distro}-daily-{stream}' + +#-------------------------------- +# job templates +#-------------------------------- +- job-template: + name: 'xci-{scenario}-{pod}-{distro}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-os-.*?-{pod}-daily-.*' + block-level: 'NODE' + + parameters: + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - label: + name: SLAVE_LABEL + default: '{slave-label}' + + triggers: + - '{auto-trigger-name}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - trigger-builds: + - project: 'xci-provision-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + - trigger-builds: + - project: 'xci-deploy-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + - trigger-builds: + - project: 'xci-functest-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + + publishers: + - email: + recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com + +- job-template: + name: 'xci-{phase}-{pod}-{distro}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate-default + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'xci-provision-.*?-{pod}-daily-.*' + - 'xci-deploy-.*?-{pod}-daily-.*' + - 'xci-functest-.*?-{pod}-daily-.*' + block-level: 'NODE' + + parameters: + - string: + name: PROJECT + default: '{project}' + - string: + name: PROJECT_REPO + default: '{project-repo}' + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - string: + name: CLONE_LOCATION + default: '{clone-location}' + - string: + name: DISTRO + default: '{distro}' + - string: + name: DIB_OS_RELEASE + default: '{dib-os-release}' + - string: + name: DIB_OS_ELEMENT + default: '{dib-os-element}' + - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: + name: DIB_OS_PACKAGES + default: '{dib-os-packages}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + + scm: + - git: + url: '$PROJECT_REPO' + branches: + - 'origin/{project-branch}' + wipe-workspace: true + timeout: 15 + + builders: + - description-setter: + description: "Built on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" + - 'xci-{phase}-builder' +#--------------------------- +# builder macros +#--------------------------- +- builder: + name: xci-provision-builder + builders: + - shell: + !include-raw: ./xci-provision.sh +- builder: + name: xci-deploy-builder + builders: + - shell: + !include-raw: ./xci-deploy.sh +- builder: + name: xci-functest-builder + builders: + - shell: + !include-raw: ./xci-functest.sh diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh new file mode 100755 index 000000000..b97b24e47 --- /dev/null +++ b/jjb/xci/xci-deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Deployment via openstack-ansible" diff --git a/jjb/xci/xci-functest.sh b/jjb/xci/xci-functest.sh new file mode 100755 index 000000000..0f58dfefc --- /dev/null +++ b/jjb/xci/xci-functest.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Functional testing with functest" diff --git a/jjb/xci/xci-provision.sh b/jjb/xci/xci-provision.sh new file mode 100755 index 000000000..b77c79caa --- /dev/null +++ b/jjb/xci/xci-provision.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Provisioning via bifrost" -- cgit 1.2.3-korg