diff options
Diffstat (limited to 'jjb')
| -rwxr-xr-x | jjb/apex/apex-build.sh | 8 | ||||
| -rwxr-xr-x | jjb/apex/apex-deploy.sh | 16 | ||||
| -rw-r--r-- | jjb/apex/apex-snapshot-deploy.sh | 2 | ||||
| -rwxr-xr-x | jjb/apex/apex-unit-test.sh | 2 | ||||
| -rw-r--r-- | jjb/cperf/cperf-ci-jobs.yml | 20 | ||||
| -rw-r--r-- | jjb/daisy4nfv/daisy-daily-jobs.yml | 1 | ||||
| -rw-r--r-- | jjb/daisy4nfv/daisy-project-jobs.yml | 11 | ||||
| -rw-r--r-- | jjb/daisy4nfv/daisy4nfv-merge-jobs.yml | 5 | ||||
| -rw-r--r-- | jjb/daisy4nfv/daisy4nfv-verify-jobs.yml | 46 | ||||
| -rw-r--r-- | jjb/dovetail/dovetail-ci-jobs.yml | 4 | ||||
| -rw-r--r-- | jjb/dovetail/dovetail-weekly-jobs.yml | 4 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.sh | 18 | ||||
| -rw-r--r-- | jjb/test-requirements.txt | 1 | ||||
| -rw-r--r-- | jjb/xci/bifrost-periodic-jobs.yml | 146 | ||||
| -rwxr-xr-x | jjb/xci/bifrost-provision.sh | 107 | ||||
| -rw-r--r-- | jjb/xci/xci-daily-jobs.yml | 169 | ||||
| -rwxr-xr-x | jjb/xci/xci-deploy.sh | 94 | ||||
| -rwxr-xr-x | jjb/yardstick/yardstick-daily.sh | 2 |
18 files changed, 387 insertions, 269 deletions
diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh index 220d02435..b6b2f212a 100755 --- a/jjb/apex/apex-build.sh +++ b/jjb/apex/apex-build.sh @@ -28,10 +28,10 @@ cd $WORKSPACE/ci ./build.sh $BUILD_ARGS RPM_VERSION=$(grep Version: $WORKSPACE/build/rpm_specs/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') # list the contents of BUILD_OUTPUT directory -echo "Build Directory is ${BUILD_DIRECTORY}" +echo "Build Directory is ${BUILD_DIRECTORY}/../.build" echo "Build Directory Contents:" echo "-------------------------" -ls -al $BUILD_DIRECTORY +ls -al ${BUILD_DIRECTORY}/../.build # list the contents of CACHE directory echo "Cache Directory is ${CACHE_DIRECTORY}" @@ -47,10 +47,10 @@ if ! echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" - echo "OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)" + echo "OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)" echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm" echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm" - echo "OPNFV_RPM_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/noarch/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)" + echo "OPNFV_RPM_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/noarch/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)" echo "OPNFV_BUILD_URL=$BUILD_URL" ) > $WORKSPACE/opnfv.properties fi diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index 1e04c1f95..4b64fd14f 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -93,8 +93,8 @@ if [[ "$BUILD_DIRECTORY" == *verify* || "$BUILD_DIRECTORY" == *promote* ]]; then DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy" NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network" DEPLOY_CMD="$(pwd)/deploy.sh" - RESOURCES="${WORKSPACE}/.build/" - CONFIG="${WORKSPACE}/build" + IMAGES="${WORKSPACE}/.build/" + BASE="${WORKSPACE}/build" LIB="${WORKSPACE}/lib" # Make sure python34 deps are installed for dep_pkg in epel-release python34 python34-PyYAML python34-setuptools; do @@ -129,7 +129,7 @@ if [[ "$BUILD_DIRECTORY" == *verify* || "$BUILD_DIRECTORY" == *promote* ]]; then # use RPMs else # find version of RPM - VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | grep -Eo '[0-9]+\.[0-9]+-[0-9]{8}') + VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | grep -Eo '[0-9]+\.[0-9]+-([0-9]{8}|[a-z]+-[0-9]\.[0-9]+)') # build RPM List which already includes base Apex RPM for pkg in ${APEX_PKGS}; do RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}.noarch.rpm" @@ -151,13 +151,13 @@ else DEPLOY_CMD=opnfv-deploy DEPLOY_SETTINGS_DIR="/etc/opnfv-apex/" NETWORK_SETTINGS_DIR="/etc/opnfv-apex/" - RESOURCES="/var/opt/opnfv/images" - CONFIG="/var/opt/opnfv" + IMAGES="/var/opt/opnfv/images" + BASE="/var/opt/opnfv" LIB="/var/opt/opnfv/lib" fi # set env vars to deploy cmd -DEPLOY_CMD="CONFIG=${CONFIG} RESOURCES=${RESOURCES} LIB=${LIB} ${DEPLOY_CMD}" +DEPLOY_CMD="BASE=${BASE} IMAGES=${IMAGES} LIB=${LIB} ${DEPLOY_CMD}" if [ "$OPNFV_CLEAN" == 'yes' ]; then if sudo test -e '/root/inventory/pod_settings.yaml'; then @@ -166,9 +166,9 @@ if [ "$OPNFV_CLEAN" == 'yes' ]; then clean_opts='' fi if [[ "$BUILD_DIRECTORY" == *verify* || "$BUILD_DIRECTORY" == *promote* ]]; then - sudo CONFIG=${CONFIG} LIB=${LIB} ./clean.sh ${clean_opts} + sudo BASE=${BASE} LIB=${LIB} ./clean.sh ${clean_opts} else - sudo CONFIG=${CONFIG} LIB=${LIB} opnfv-clean ${clean_opts} + sudo BASE=${BASE} LIB=${LIB} opnfv-clean ${clean_opts} fi fi diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh index 8274740c8..06c002319 100644 --- a/jjb/apex/apex-snapshot-deploy.sh +++ b/jjb/apex/apex-snapshot-deploy.sh @@ -22,7 +22,7 @@ echo echo "Cleaning server" pushd ci > /dev/null -sudo CONFIG=../build/ LIB=../lib ./clean.sh +sudo BASE=../build/ LIB=../lib ./clean.sh popd > /dev/null echo "Downloading latest snapshot properties file" diff --git a/jjb/apex/apex-unit-test.sh b/jjb/apex/apex-unit-test.sh index 5c4341741..12cb862b0 100755 --- a/jjb/apex/apex-unit-test.sh +++ b/jjb/apex/apex-unit-test.sh @@ -9,7 +9,7 @@ echo pushd ci/ > /dev/null -sudo CONFIG="${WORKSPACE}/build" LIB="${WORKSPACE}/lib" ./clean.sh +sudo BASE="${WORKSPACE}/build" LIB="${WORKSPACE}/lib" ./clean.sh ./test.sh popd diff --git a/jjb/cperf/cperf-ci-jobs.yml b/jjb/cperf/cperf-ci-jobs.yml index 2742f08f7..f6e068530 100644 --- a/jjb/cperf/cperf-ci-jobs.yml +++ b/jjb/cperf/cperf-ci-jobs.yml @@ -126,14 +126,20 @@ undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) - sudo scp $INSTALLER_IP:/home/stack/stackrc /tmp/stackrc - source /tmp/stackrc + + sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/overcloudrc /tmp/overcloudrc + sudo chmod 755 /tmp/overcloudrc + source /tmp/overcloudrc # robot suites need the ssh key to log in to controller nodes, so throwing it # in tmp, and mounting /tmp as $HOME as far as robot is concerned - sudo mkdir -p /tmp/.ssh - sudo scp $INSTALLER_IP:/home/stack/.ssh/id_rsa /tmp/.ssh/ - sudo chmod -R 0600 /tmp/.ssh + sudo rm -rf /tmp/.ssh + sudo mkdir /tmp/.ssh + sudo chmod 0700 /tmp/.ssh + sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/.ssh/id_rsa /tmp/.ssh/ + sudo chown -R jenkins-ci:jenkins-ci /tmp/.ssh + # done with sudo. jenkins-ci is the user from this point + chmod 0600 /tmp/.ssh/id_rsa # cbench requires the openflow drop test feature to be installed. sshpass -p karaf ssh -o StrictHostKeyChecking=no \ @@ -144,7 +150,7 @@ docker pull opnfv/cperf:$DOCKER_TAG - robot_cmd="pybot -e exclude -L TRACE \ + robot_cmd="pybot -e exclude -L TRACE -d /tmp \ -v ODL_SYSTEM_1_IP:${SDN_CONTROLLER_IP} \ -v ODL_SYSTEM_IP:${SDN_CONTROLLER_IP} \ -v BUNDLEFOLDER:/opt/opendaylight \ @@ -156,7 +162,7 @@ -v of_port:6653" robot_suite="/home/opnfv/repos/odl_test/csit/suites/openflowplugin/Performance/010_Cbench.robot" - docker run -v /tmp:/tmp opnfv/cperf:$DOCKER_TAG ${robot_cmd} ${robot_suite} + docker run -ti -v /tmp:/tmp opnfv/cperf:$DOCKER_TAG ${robot_cmd} ${robot_suite} - builder: name: cperf-cleanup diff --git a/jjb/daisy4nfv/daisy-daily-jobs.yml b/jjb/daisy4nfv/daisy-daily-jobs.yml index c5d8e7e8b..aac76baa4 100644 --- a/jjb/daisy4nfv/daisy-daily-jobs.yml +++ b/jjb/daisy4nfv/daisy-daily-jobs.yml @@ -71,7 +71,6 @@ use-build-blocker: true blocking-jobs: - 'daisy-daily-.*' - - 'daisy4nfv-(merge|verify)-.*' block-level: 'NODE' wrappers: diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index 52769ca88..e631ee9b9 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -54,6 +54,11 @@ enabled: true max-total: 4 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{installer}-daily-.*' + block-level: 'NODE' scm: - git-scm @@ -138,12 +143,6 @@ enabled: true max-total: 6 option: 'project' - - build-blocker: - use-build-blocker: true - blocking-jobs: - - '{installer}-daily-.*' - - 'daisy4nfv-(merge|verify)-.*' - block-level: 'NODE' scm: - git-scm diff --git a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml index 95e72e550..9e7b867af 100644 --- a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml @@ -29,7 +29,7 @@ - 'build': slave-label: 'opnfv-build-centos' - 'deploy-virtual': - slave-label: 'opnfv-build-centos' + slave-label: 'daisy-virtual' ##################################### # jobs ##################################### @@ -148,8 +148,7 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - '{alias}-(merge|verify)-.*' - - '{project}-daily-.*' + - '{alias}-merge-(master|danube)' block-level: 'NODE' scm: diff --git a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml index 9f44d99fb..a0ec2ebd7 100644 --- a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml @@ -1,10 +1,7 @@ - project: name: 'daisy4nfv-verify-jobs' - project: 'daisy' - installer: 'daisy' - ########################################################## # use alias to keep the jobs'name existed alread unchanged ########################################################## @@ -26,7 +23,9 @@ # patch verification phases ##################################### phase: - - 'build': + - unit: + slave-label: 'opnfv-build' + - build: slave-label: 'opnfv-build-centos' ##################################### # jobs @@ -39,29 +38,22 @@ ##################################### - job-template: name: '{alias}-verify-{stream}' - project-type: multijob - disabled: false - concurrent: true - properties: - logrotate-default - throttle: enabled: true max-total: 4 option: 'project' - scm: - git-scm - wrappers: - ssh-agent-wrapper - timeout: timeout: 360 fail: true - triggers: - gerrit: server-name: 'gerrit.opnfv.org' @@ -108,6 +100,15 @@ - description-setter: description: "Built on $NODE_NAME" - multijob: + name: unit + condition: SUCCESSFUL + projects: + - name: '{alias}-verify-{name}-{stream}' + current-parameters: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: name: build condition: SUCCESSFUL projects: @@ -124,33 +125,21 @@ - job-template: name: '{alias}-verify-{phase}-{stream}' - disabled: '{obj:disabled}' - concurrent: true - properties: - logrotate-default - throttle: enabled: true max-total: 6 option: 'project' - - build-blocker: - use-build-blocker: true - blocking-jobs: - - '{alias}-(merge|verify)-.*' - - '{installer}-daily-.*' - block-level: 'NODE' - scm: - git-scm - wrappers: - ssh-agent-wrapper - timeout: timeout: 360 fail: true - parameters: - project-parameter: project: '{project}' @@ -158,7 +147,6 @@ - '{slave-label}-defaults' - '{alias}-verify-defaults': gs-pathname: '{gs-pathname}' - builders: - description-setter: description: "Built on $NODE_NAME" @@ -177,6 +165,16 @@ - shell: !include-raw: ./daisy4nfv-workspace-cleanup.sh +- builder: + name: daisy-verify-unit-macro + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace + tox -e py27 + ##################################### # parameter macros ##################################### diff --git a/jjb/dovetail/dovetail-ci-jobs.yml b/jjb/dovetail/dovetail-ci-jobs.yml index b65e6d5ef..5651fc3f2 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yml +++ b/jjb/dovetail/dovetail-ci-jobs.yml @@ -169,6 +169,7 @@ - timeout: timeout: 180 abort: true + - fix-workspace-permissions triggers: - '{auto-trigger-name}' @@ -208,9 +209,6 @@ - 'dovetail-cleanup' - 'dovetail-run' - wrappers: - - fix-workspace-permissions - publishers: - archive: artifacts: 'results/**/*' diff --git a/jjb/dovetail/dovetail-weekly-jobs.yml b/jjb/dovetail/dovetail-weekly-jobs.yml index 7b3ede902..eaa11b54a 100644 --- a/jjb/dovetail/dovetail-weekly-jobs.yml +++ b/jjb/dovetail/dovetail-weekly-jobs.yml @@ -78,6 +78,7 @@ - timeout: timeout: '{job-timeout}' abort: true + - fix-workspace-permissions parameters: - project-parameter: @@ -114,9 +115,6 @@ - 'dovetail-cleanup' - 'dovetail-run' - wrappers: - - fix-workspace-permissions - publishers: - archive: artifacts: 'results/**/*' diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index 9bd711bc6..5d73a9d70 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -17,14 +17,16 @@ echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..." echo "--------------------------------------------------------" echo - -if [[ -n $(ps -ef|grep 'docker build'|grep -v grep) ]]; then - echo "There is already another build process in progress:" - echo $(ps -ef|grep 'docker build'|grep -v grep) - # Abort this job since it will collide and might mess up the current one. - echo "Aborting..." - exit 1 -fi +count=30 # docker build jobs might take up to ~30 min +while [[ -n `ps -ef|grep 'docker build'|grep -v grep` ]]; do + echo "Build in progress. Waiting..." + sleep 60 + count=$(( $count - 1 )) + if [ $count -eq 0 ]; then + echo "Timeout. Aborting..." + exit 1 + fi +done # Remove previous running containers if exist if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then diff --git a/jjb/test-requirements.txt b/jjb/test-requirements.txt new file mode 100644 index 000000000..6b700dcfc --- /dev/null +++ b/jjb/test-requirements.txt @@ -0,0 +1 @@ +jenkins-job-builder diff --git a/jjb/xci/bifrost-periodic-jobs.yml b/jjb/xci/bifrost-periodic-jobs.yml new file mode 100644 index 000000000..31aa8dade --- /dev/null +++ b/jjb/xci/bifrost-periodic-jobs.yml @@ -0,0 +1,146 @@ +- project: + project: 'releng' + + name: 'bifrost-periodic' +#-------------------------------- +# Branch Anchors +#-------------------------------- +# the versions stated here default to branches which then later +# on used for checking out the branches, pulling in head of the branch. + master: &master + stream: master + openstack-bifrost-version: '{stream}' + opnfv-releng-version: 'master' + gs-pathname: '' + ocata: &ocata + stream: ocata + openstack-bifrost-version: 'stable/{stream}' + opnfv-releng-version: 'master' + gs-pathname: '/{stream}' +#-------------------------------- +# XCI PODs +#-------------------------------- + pod: + - virtual: + <<: *master + - virtual: + <<: *ocata +#-------------------------------- +# XCI PODs +#-------------------------------- +#-------------------------------- +# Supported Distros +#-------------------------------- + distro: + - 'xenial': + disabled: false + slave-label: xci-xenial-virtual + dib-os-release: 'xenial' + dib-os-element: 'ubuntu-minimal' + dib-os-packages: 'vlan,vim,less,bridge-utils,sudo,language-pack-en,iputils-ping,rsyslog,curl,python,debootstrap,ifenslave,ifenslave-2.6,lsof,lvm2,tcpdump,nfs-kernel-server,chrony,iptables' + extra-dib-elements: 'openssh-server' + - 'centos7': + disabled: true + slave-label: xci-centos7-virtual + dib-os-release: '7' + dib-os-element: 'centos7' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + - 'suse': + disabled: true + slave-label: xci-suse-virtual + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + +#-------------------------------- +# jobs +#-------------------------------- + jobs: + - 'bifrost-provision-{pod}-{distro}-periodic-{stream}' + +#-------------------------------- +# job templates +#-------------------------------- +- job-template: + name: 'bifrost-provision-{pod}-{distro}-periodic-{stream}' + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '^xci-os.*' + - '^xci-deploy.*' + - '^xci-functest.*' + - '^bifrost-periodic-.*' + - '^osa-periodic-.*' + block-level: 'NODE' + - logrotate-default + + parameters: + - project-parameter: + project: '{project}' + branch: '{opnfv-releng-version}' + - string: + name: XCI_FLAVOR + default: 'ha' + - string: + name: OPENSTACK_BIFROST_VERSION + default: '{openstack-bifrost-version}' + - string: + name: OPNFV_RELENG_VERSION + default: '{opnfv-releng-version}' + - string: + name: DISTRO + default: '{distro}' + - string: + name: DIB_OS_RELEASE + default: '{dib-os-release}' + - string: + name: DIB_OS_ELEMENT + default: '{dib-os-element}' + - string: + name: DIB_OS_PACKAGES + default: '{dib-os-packages}' + - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - label: + name: SLAVE_LABEL + default: '{slave-label}' + - string: + name: ANSIBLE_VERBOSITY + default: '' + + wrappers: + - fix-workspace-permissions + + scm: + - git-scm + + # trigger is disabled until we know which jobs we will have + # and adjust stuff accordingly + triggers: + - timed: '#@midnight' + + builders: + - description-setter: + description: "Built on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" + - 'bifrost-provision-builder' + +#--------------------------- +# builder macros +#--------------------------- +- builder: + name: bifrost-provision-builder + builders: + - shell: + !include-raw: ./bifrost-provision.sh diff --git a/jjb/xci/bifrost-provision.sh b/jjb/xci/bifrost-provision.sh new file mode 100755 index 000000000..d8e17c463 --- /dev/null +++ b/jjb/xci/bifrost-provision.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +trap cleanup_and_upload EXIT + +function fix_ownership() { + if [ -z "${JOB_URL+x}" ]; then + echo "Not running as part of Jenkins. Handle the logs manually." + else + # Make sure cache exists + [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache + + sudo chown -R jenkins:jenkins $WORKSPACE + sudo chown -R jenkins:jenkins ${HOME}/.cache + fi +} + +function cleanup_and_upload() { + original_exit=$? + fix_ownership + exit $original_exit +} + +# check distro to see if we support it +if [[ ! "$DISTRO" =~ (xenial|centos7|suse) ]]; then + echo "Distro $DISTRO is not supported!" + exit 1 +fi + +# remove previously cloned repos +sudo /bin/rm -rf /opt/bifrost /opt/openstack-ansible /opt/releng /opt/functest + +# Fix up permissions +fix_ownership + +# ensure the versions to checkout are set +export OPENSTACK_BIFROST_VERSION=${OPENSTACK_BIFROST_VERSION:-master} +export OPNFV_RELENG_VERSION=${OPNFV_RELENG_VERSION:-master} + +# log some info +echo -e "\n" +echo "***********************************************************************" +echo "* *" +echo "* Provision OpenStack Nodes *" +echo "* *" +echo " bifrost version: $OPENSTACK_BIFROST_VERSION" +echo " releng version: $OPNFV_RELENG_VERSION" +echo "* *" +echo "***********************************************************************" +echo -e "\n" + +# clone the repos and checkout the versions +sudo git clone --quiet https://git.openstack.org/openstack/bifrost /opt/bifrost +cd /opt/bifrost && sudo git checkout --quiet $OPENSTACK_BIFROST_VERSION +echo "xci: using bifrost commit" +git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>' + +sudo git clone --quiet https://gerrit.opnfv.org/gerrit/releng /opt/releng +cd /opt/releng && sudo git checkout --quiet $OPNFV_RELENG_VERSION +echo "xci: using releng commit" +git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>' + +# combine opnfv and upstream scripts/playbooks +sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/ + +# cleanup remnants of previous deployment +cd /opt/bifrost +sudo -E ./scripts/destroy-env.sh + +# provision 6 VMs; xcimaster, controller00, controller01, controller02, compute00, and compute01 +cd /opt/bifrost +sudo -E ./scripts/bifrost-provision.sh + +# list the provisioned VMs +cd /opt/bifrost +source env-vars +ironic node-list +virsh list + + +echo "OpenStack nodes are provisioned!" +# here we have to do something in order to capture what was the working sha1 +# hardcoding stuff for the timebeing + +cd /opt/bifrost +BIFROST_GIT_SHA1=$(git rev-parse HEAD) + +# log some info +echo -e "\n" +echo "***********************************************************************" +echo "* BIFROST SHA1 TO PIN *" +echo "* *" +echo " $BIFROST_GIT_SHA1" +echo "* *" +echo "***********************************************************************" + +echo -e "\n" diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml index dbe3b654b..f9fbc1de8 100644 --- a/jjb/xci/xci-daily-jobs.yml +++ b/jjb/xci/xci-daily-jobs.yml @@ -1,34 +1,37 @@ +#-------------------------------- +# These jobs run on a daily basis and deploy OpenStack +# using the pinned versions of opnfv/releng, openstack/bifrost +# and openstack/openstack-ansible. Due to this, there is no +# version/branch is set/passed to jobs and instead the versions +# are checked out based on what is configured. +#-------------------------------- - project: - name: 'bifrost-osa-daily' + project: 'releng' + + name: 'xci-daily' #-------------------------------- -# BRANCH ANCHORS +# Branch Anchors #-------------------------------- -# the versions stated here default to branches which then later -# on used for checking out the branches, pulling in head of the branch. -# but they also allow us to state sha1 so instead of checking out the -# branches, we can check out sha1 if we want to use locked/specific -# sha1 or manually enter sha1. master: &master stream: master - openstack-osa-version: '{stream}' - openstack-bifrost-version: '{stream}' - opnfv-releng-version: 'master' + opnfv-releng-version: master gs-pathname: '' ocata: &ocata stream: ocata - openstack-osa-version: 'stable/{stream}' - openstack-bifrost-version: 'stable/{stream}' - opnfv-releng-version: 'master' + opnfv-releng-version: master gs-pathname: '/{stream}' #-------------------------------- -# scenarios +# Scenarios #-------------------------------- scenario: - # HA scenarios - 'os-nosdn-nofeature-ha': auto-trigger-name: 'daily-trigger-disabled' + xci-flavor: 'ha' + - 'os-nosdn-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + xci-flavor: 'noha' #-------------------------------- -# XCI PODs +# XCI PODs #-------------------------------- pod: - virtual: @@ -36,7 +39,7 @@ - virtual: <<: *ocata #-------------------------------- -# Supported Distros +# Supported Distros #-------------------------------- distro: - 'xenial': @@ -44,7 +47,7 @@ slave-label: xci-xenial-virtual dib-os-release: 'xenial' dib-os-element: 'ubuntu-minimal' - dib-os-packages: 'vlan,vim,less,bridge-utils,sudo,language-pack-en,iputils-ping,rsyslog,curl,python,debootstrap,ifenslave,ifenslave-2.6,lsof,lvm2,tcpdump,nfs-kernel-server,chrony' + dib-os-packages: 'vlan,vim,less,bridge-utils,sudo,language-pack-en,iputils-ping,rsyslog,curl,python,debootstrap,ifenslave,ifenslave-2.6,lsof,lvm2,tcpdump,nfs-kernel-server,chrony,iptabls' extra-dib-elements: 'openssh-server' - 'centos7': disabled: true @@ -65,7 +68,6 @@ # Phases #-------------------------------- phase: - - 'provision' - 'deploy' - 'functest' #-------------------------------- @@ -76,19 +78,6 @@ - 'xci-{phase}-{pod}-{distro}-daily-{stream}' #-------------------------------- -# VM defaults -#-------------------------------- -- defaults: - name: daily_vm_defaults - test-vm-num-nodes: '6' - test-vm-node-names: 'xcimaster controller00 controller01 controller02 compute00 compute01' - vm-domain-type: 'kvm' - vm-cpu: '8' - vm-disk: '100' - vm-memory-size: '16384' - vm-disk-cache: 'unsafe' - -#-------------------------------- # job templates #-------------------------------- - job-template: @@ -103,25 +92,20 @@ use-build-blocker: true blocking-jobs: - '^xci-os.*' + - '^xci-deploy.*' + - '^xci-functest.*' + - '^bifrost-periodic-.*' + - '^osa-periodic-.*' block-level: 'NODE' - logrotate-default parameters: - string: - name: OPENSTACK_OSA_VERSION - default: '{openstack-osa-version}' - - string: - name: OPENSTACK_BIFROST_VERSION - default: '{openstack-osa-version}' - - string: - name: OPNFV_RELENG_VERSION - default: '{opnfv-releng-version}' - - string: - name: USE_PROMOTED_VERSIONS - default: 'true' - - string: name: DEPLOY_SCENARIO default: '{scenario}' + - string: + name: XCI_FLAVOR + default: '{xci-flavor}' - label: name: SLAVE_LABEL default: '{slave-label}' @@ -130,42 +114,25 @@ - '{auto-trigger-name}' wrappers: - - xci-fix-perms-workspace + - fix-workspace-permissions builders: - description-setter: description: "Built on $NODE_NAME" - trigger-builds: - - project: 'xci-provision-{pod}-{distro}-daily-{stream}' - current-parameters: false - predefined-parameters: | - OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION - OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION - OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION - USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - same-node: true - block: true - - trigger-builds: - project: 'xci-deploy-{pod}-{distro}-daily-{stream}' current-parameters: false predefined-parameters: | - OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION - OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION - OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION - USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS DEPLOY_SCENARIO=$DEPLOY_SCENARIO + XCI_FLAVOR=$XCI_FLAVOR same-node: true block: true - trigger-builds: - project: 'xci-functest-{pod}-{distro}-daily-{stream}' current-parameters: false predefined-parameters: | - OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION - OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION - OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION - USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS DEPLOY_SCENARIO=$DEPLOY_SCENARIO + XCI_FLAVOR=$XCI_FLAVOR same-node: true block: true block-thresholds: @@ -182,37 +149,36 @@ disabled: '{obj:disabled}' - defaults: daily_vm_defaults - concurrent: false properties: - build-blocker: use-build-blocker: true blocking-jobs: - - '^xci-provision.*' - '^xci-deploy.*' - '^xci-functest.*' + - '^bifrost-periodic-.*' + - '^osa-periodic-.*' block-level: 'NODE' - logrotate-default + wrappers: + - fix-workspace-permissions + + scm: + - git-scm + parameters: - - string: - name: OPENSTACK_OSA_VERSION - default: '{openstack-osa-version}' - - string: - name: OPENSTACK_BIFROST_VERSION - default: '{openstack-osa-version}' - - string: - name: OPNFV_RELENG_VERSION - default: '{opnfv-releng-version}' - - string: - name: USE_PROMOTED_VERSIONS - default: 'true' + - project-parameter: + project: '{project}' + branch: '{opnfv-releng-version}' - string: name: DEPLOY_SCENARIO default: 'os-nosdn-nofeature-ha' - string: + name: XCI_FLAVOR + default: 'ha' + - string: name: DISTRO default: '{distro}' - string: @@ -222,32 +188,11 @@ name: DIB_OS_ELEMENT default: '{dib-os-element}' - string: - name: EXTRA_DIB_ELEMENTS - default: '{extra-dib-elements}' - - string: name: DIB_OS_PACKAGES default: '{dib-os-packages}' - string: - name: TEST_VM_NUM_NODES - default: '{test-vm-num-nodes}' - - string: - name: TEST_VM_NODE_NAMES - default: '{test-vm-node-names}' - - string: - name: VM_DOMAIN_TYPE - default: '{vm-domain-type}' - - string: - name: VM_CPU - default: '{vm-cpu}' - - string: - name: VM_DISK - default: '{vm-disk}' - - string: - name: VM_MEMORY_SIZE - default: '{vm-memory-size}' - - string: - name: VM_DISK_CACHE - default: '{vm-disk-cache}' + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' - string: name: CLEAN_DIB_IMAGES default: 'true' @@ -258,38 +203,20 @@ name: ANSIBLE_VERBOSITY default: '' - wrappers: - - xci-fix-perms-workspace - builders: - description-setter: description: "Built on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" - 'xci-{phase}-builder' #--------------------------- -# wrapper macros -#--------------------------- -- wrapper: - name: xci-fix-perms-workspace - wrappers: - - pre-scm-buildstep: - - shell: | - #!/bin/bash - sudo chown -R $USER $WORKSPACE || exit 1 - -#--------------------------- # builder macros #--------------------------- - builder: - name: xci-provision-builder - builders: - - shell: - !include-raw: ./xci-provision.sh -- builder: name: xci-deploy-builder builders: - shell: !include-raw: ./xci-deploy.sh + - builder: name: xci-functest-builder builders: diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh index 87f9ec8db..cf5fe32d3 100755 --- a/jjb/xci/xci-deploy.sh +++ b/jjb/xci/xci-deploy.sh @@ -11,83 +11,21 @@ set -o errexit set -o nounset set -o pipefail -trap cleanup_and_upload EXIT - -function fix_ownership() { - if [ -z "${JOB_URL+x}" ]; then - echo "Not running as part of Jenkins. Handle the logs manually." - else - # Make sure cache exists - [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache - - sudo chown -R jenkins:jenkins $WORKSPACE - sudo chown -R jenkins:jenkins ${HOME}/.cache - fi -} - -function cleanup_and_upload() { - original_exit=$? - fix_ownership - exit $original_exit -} - -# check distro to see if we support it -if [[ ! "$DISTRO" =~ (xenial|centos7|suse) ]]; then - echo "Distro $DISTRO is not supported!" - exit 1 +cd $WORKSPACE/prototypes/xci + +# for daily jobs, we want to use working versions +# for periodic jobs, we will use whatever is set in the job, probably master +if [[ "$JOB_NAME" =~ "daily" ]]; then + # source pinned-vars to get releng version + source ./config/pinned-versions + + # checkout the version + git checkout -q $OPNFV_RELENG_VERSION + echo "Info: Using $OPNFV_RELENG_VERSION" +elif [[ "$JOB_NAME" =~ "periodic" ]]; then + echo "Info: Using $OPNFV_RELENG_VERSION" fi -# remove previously cloned repos -sudo /bin/rm -rf /opt/openstack-ansible /opt/stack /opt/releng /opt/functest - -# Fix up permissions -fix_ownership - -# openstack-ansible enables strict host key checking by default -export ANSIBLE_HOST_KEY_CHECKING=False - -# ensure the versions to checkout are set -export OPENSTACK_OSA_VERSION=${OPENSTACK_OSA_VERSION:-master} -export OPNFV_RELENG_VERSION=${OPNFV_RELENG_VERSION:-master} - -# log some info -echo -e "\n" -echo "***********************************************************************" -echo "* *" -echo "* Deploy OpenStack *" -echo "* *" -echo " openstack-ansible version: $OPENSTACK_OSA_VERSION" -echo " releng version: $OPNFV_RELENG_VERSION" -echo "* *" -echo "***********************************************************************" -echo -e "\n" -# clone releng repo -sudo git clone --quiet https://gerrit.opnfv.org/gerrit/releng /opt/releng -cd /opt/releng && sudo git checkout --quiet $OPNFV_RELENG_VERSION -echo "xci: using openstack-ansible commit" -git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>' - -# display the nodes -echo "xci: OpenStack nodes" -cd /opt/bifrost -source env-vars -ironic node-list - -# this script will be reused for promoting openstack-ansible versions and using -# promoted openstack-ansible versions as part of xci daily. -USE_PROMOTED_VERSIONS=${USE_PROMOTED_VERSIONS:-false} -if [ $USE_PROMOTED_VERSIONS = "true" ]; then - echo "TBD: Will use the promoted versions of openstack/opnfv projects" -fi - -cd /opt/releng/prototypes/openstack-ansible/scripts -sudo -E ./osa-deploy.sh - -# log some info -echo -e "\n" -echo "***********************************************************************" -echo "* *" -echo "* OpenStack deployment is completed! *" -echo "* *" -echo "***********************************************************************" -echo -e "\n" +# proceed with the deployment +cd $WORKSPACE/prototypes/xci +sudo -E ./xci-deploy.sh diff --git a/jjb/yardstick/yardstick-daily.sh b/jjb/yardstick/yardstick-daily.sh index f769e9cdd..973f83ad5 100755 --- a/jjb/yardstick/yardstick-daily.sh +++ b/jjb/yardstick/yardstick-daily.sh @@ -18,7 +18,7 @@ if [[ ${INSTALLER_TYPE} == 'apex' ]]; then elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then # If production lab then creds may be retrieved dynamically # creds are on the jumphost, always in the same folder - labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/openrc" + labconfig="-v $LAB_CONFIG/admin-openrc:/etc/yardstick/openstack.creds" # If dev lab, credentials may not be the default ones, just provide a path to put them into docker # replace the default one by the customized one provided by jenkins config fi |