From 1fe93bf666765927d9ed15ead14db85caf51098c Mon Sep 17 00:00:00 2001 From: Thomas F Herbert Date: Mon, 27 Jun 2016 10:37:46 -0400 Subject: RPM: Apex: Refactor build scripts: Remove reference to undercloud. Fix dependencies list. Refactor build ovs and build dpdk and test scripts. Correct default dpdk version in build ovs script. Remove references to instack VM -- no longer necessary. Remove script using custom Apex-lite-liberty for testing Consolidated functions and cleanup in response to reviewers. Change-Id: I0875acd87580769c7cd7e87b97bb3e04f0159b32 Signed-off-by: Thomas F Herbert --- build/BuildAndTestOVS.sh | 87 ++++++---- build/build_dpdk_rpm.sh | 15 +- build/build_ovs_rpm.sh | 82 +++++----- build/functions.sh | 42 +++++ build/instack_ovs.sh | 401 ----------------------------------------------- build/test_ovs_rpm.sh | 25 ++- ci/clean.sh | 71 +++++---- 7 files changed, 193 insertions(+), 530 deletions(-) create mode 100755 build/functions.sh delete mode 100755 build/instack_ovs.sh diff --git a/build/BuildAndTestOVS.sh b/build/BuildAndTestOVS.sh index 10a2f05..f12ab3e 100755 --- a/build/BuildAndTestOVS.sh +++ b/build/BuildAndTestOVS.sh @@ -72,8 +72,26 @@ while getopts "a:dg:hi:p:tu:v" opt; do esac done -if [ -z $TAG ]; then - TAG=master +# +# Default Config options +# +echo =============================================== +echo Default Configuration Options. +echo =============================================== +echo option NOCHECK is set to $NOCHECK +echo build DPDK option is set to $DPDK +echo DPDK Patch URL is set to $DPDK_PATCH +echo DPDK Version is set to $DPDK_VERSION +echo Option for OVS Kernel Module is set to $KMOD +echo =============================================== +if [[ $NOCHECK =~ "yes" ]]; then + setnocheck="-c" +fi +if [[ $KMOD =~ "yes" ]]; then + setkmod="-k" +fi +if [[ $DPDK =~ "yes" ]]; then + setdpdk="-d" fi if [ -z $OVS_REPO_URL ]; then @@ -89,33 +107,24 @@ else echo Will use default kernel in ovs test vm fi +if [ -z $TAG ]; then + TAG=master +fi + if [ ! -z $DPDK ]; then setbuilddpdk="-d" fi if [ -z ${WORKSPACE+1} ]; then # We are not being run by Jenkins. - export WORKSPACE=$HOME/opnfv/ovsnfv - mkdir -p opnfv - cd opnfv - git clone https://git.opnfv.org/ovsnfv -fi - -export BUILD_BASE=$WORKSPACE/build - - - -if [ ! -d $BUILD_BASE ] -then - mkdir -p $BUILD_BASE + export WORKSPACE=`pwd` fi -if [ ! -f $BUILD_BASE/config ]; then - touch $BUILD_BASE/config +if [ -z ${BUILD_BASE+1} ]; then + export BUILD_BASE=$WORKSPACE fi -export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE -source $BUILD_BASE/config +export PATH=$PATH:$BUILD_BASE cd $BUILD_BASE export TOPDIR=$BUILD_BASE @@ -132,19 +141,35 @@ mkdir -p $RPMDIR/RPMS mkdir -p $RPMDIR/SOURCES mkdir -p $RPMDIR/SPECS mkdir -p $RPMDIR/SRPMS - - +# +# build dpdk rpm locally. +# +if [[ "$DPDK" =~ "yes" ]]; then + echo "===============================" + echo Build DPDK RPMs + echo + $BUILD_BASE/build_dpdk_rpm.sh -g $DPDK_VERSION +fi +# +# Build locally and copy RPMS +# +echo "===============================" +echo build OVS rpm locally +echo + ./build_ovs_rpm.sh $setnocheck -g $TAG $setdpdk $setkmod -p $OVS_PATCH -u $OVS_REPO_URL +# +# Test rpm +# if [ ! -z $TESTRPM ]; then - # Spawn VM to do the testing. - if [ ! -z $kernel_version ]; then - instack_ovs.sh -a $kernel_major -g $TAG -i $kernel_minor -p $OVS_PATCH -t -u $OVS_REPO_URL - else - instack_ovs.sh $setbuilddpdk -g $TAG -p $OVS_PATCH -t -u $OVS_REPO_URL - fi -else - # Run build locally. - build_ovs_rpm.sh $setbuilddpdk -g $TAG -p $OVS_PATCH -u $OVS_REPO_URL - cp $HOME/rpmbuild/RPMS/* $TMP_RELEASE_DIR + ./test_ovs_rpm.sh $setdpdk $setkmod fi +# +# If tests pass, copy rpms to release dir +# +echo "===============================" +echo copy rpms to release dir +echo +cp $RPMDIR/RPMS/x86_64/* $TMP_RELEASE_DIR + exit 0 diff --git a/build/build_dpdk_rpm.sh b/build/build_dpdk_rpm.sh index 1812dc3..91a7fdf 100755 --- a/build/build_dpdk_rpm.sh +++ b/build/build_dpdk_rpm.sh @@ -61,13 +61,14 @@ then rm -rf $TMPDIR fi -echo "---------------------" -echo "Install dependencies for dpdk" -echo -sudo yum -y install gcc make python-devel openssl-devel autoconf automake rpm-build \ - redhat-rpm-config libtool libpcap-devel numactl-devel python-sphinx \ - libvirt-devel - +function install_pre_reqs() { + echo "----------------------------------------" + echo Install dependencies for dpdk. + echo + sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \ + kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \ + libtool python-twisted-core desktop-file-utils groff PyQt4 +} mkdir -p $TMPDIR diff --git a/build/build_ovs_rpm.sh b/build/build_ovs_rpm.sh index b03f4ab..e28c23f 100755 --- a/build/build_ovs_rpm.sh +++ b/build/build_ovs_rpm.sh @@ -16,27 +16,11 @@ set -e echo "===============================" -echo executing $0 $@ -echo executing on machine `uname -a` +echo "executing $0 $@" +echo "executing on machine `uname -a`" usage() { - echo run BuildAndTestOVS -h for help -} - -function delrpm() { - set +e - rpm -q $1 - if [ $? -eq 0 ]; then - sudo rpm -e --allmatches $1 - fi - set -e -} -function cleanrpms() { - delrpm openvswitch - delrpm dpdk-devel - delrpm dpdk-tools - delrpm dpdk-examples - delrpm dpdk + echo "run BuildAndTestOVS -h for help" } while getopts "cdg:hkp:u:v" opt; do @@ -73,20 +57,26 @@ HOME=`pwd` TOPDIR=$HOME TMPDIR=$TOPDIR/ovsrpm +BUILDDIR=$HOME + +source $BUILDDIR/functions.sh + echo "---------------------------------------" -echo Clean out old working dir +echo "Clean out old working dir." echo if [ -d $TMPDIR ] then rm -rf $TMPDIR fi -echo "----------------------------------------" -echo Install pre-reqs. -echo -sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \ - kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \ - libtool python-twisted-core desktop-file-utils groff PyQt4 +function install_pre_reqs() { + echo "----------------------------------------" + echo "Install pre-reqs." + echo + sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \ + kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \ + libtool python-twisted-core desktop-file-utils groff PyQt4 +} VERSION=2.3.90 os_type=fedora @@ -95,7 +85,7 @@ kernel_version=$(uname -a | awk '{print $3}') RPMDIR=$HOME/rpmbuild echo "---------------------------------------" -echo Clean out old reminents of old rpms and rpm _topdir. +echo "Clean out old reminents of old rpms and rpm _topdir." echo rm openvswitch*.rpm || true @@ -104,7 +94,7 @@ if [ -d $RPMDIR ]; then fi echo "---------------------------------------" -echo Create new rpm _topdir. +echo "Create new rpm _topdir." echo mkdir -p $HOME/rpmbuild/RPMS mkdir -p $HOME/rpmbuild/SOURCES @@ -123,14 +113,15 @@ if [ ! -z $DPDK ]; then echo "----------------------------------" echo "Clone Fedora copr repo and copy files." echo - git clone https://github.com/tfherbert/ovs-snap.git + git clone https://github.com/tfherbert/ovs-snap.git cd ovs-snap git checkout $COPR_OVS_VERSION + echo "-----------------------------------" cp $TMPDIR/ovs-snap/openvswitch.spec $RPMDIR/SPECS cp $TMPDIR/ovs-snap/* $RPMDIR/SOURCES snapgit=`grep "define snapver" $TMPDIR/ovs-snap/openvswitch.spec | cut -c26-33` - echo "-------------------------------------------" - echo "Remove old dpdk, ovs and dpdk development rpms" + echo "-----------------------------------------------" + echo "Remove any old installed ovs and dpdk rpms." echo cleanrpms @@ -165,6 +156,7 @@ if [ ! -z $DPDK ]; then echo "--------------------------------------------" echo "Build openvswitch RPM" echo + rpmbuild -bb --define "_topdir `echo $RPMDIR`" $setnocheck openvswitch.spec else echo "-------------------------------------------------" echo "Build OVS without DPDK:" @@ -181,7 +173,7 @@ else export VERSION=$basever echo "--------------------------------------------" - echo making distribution tarball for Open vswitch version $VERSION + echo "Making distribution tarball for Open vswitch version $VERSION" echo ./boot.sh ./configure @@ -197,17 +189,27 @@ else curl --silent --output $HOME/rpmbuild/SOURCES/openvswitch-${VERSION}.tar.gz http://openvswitch.org/releases/openvswitch-${VERSION}.tar.gz fi - if [ ! -z $kmod ]; then - echo "--------------------------------------------" - echo "Building openvswitch kernel module RPM" - echo - rpmbuild -bb -D "kversion $kernel_version" -D "kflavors default" --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch-kmod-${os_type}.spec - fi +fi +# +# This section is for building OVS kernel module. +# +if [ ! -z $kmod ]; then + cd $TMPDIR/ovs echo "--------------------------------------------" - echo "Build openvswitch RPM" + echo "Making distribution tarball for Open vswitch version $VERSION" + echo + ./boot.sh + ./configure + make dist + echo "--------------------------------------------" + echo "Copy distribution tarball to $HOME/rpmbuild/SOURCES" + echo + cp openvswitch-*.tar.gz $HOME/rpmbuild/SOURCES + echo "--------------------------------------------" + echo "Building openvswitch kernel module RPM" echo + rpmbuild -bb -D "kversion $kernel_version" -D "kflavors default" --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch-kmod-${os_type}.spec fi -rpmbuild -bb --define "_topdir `echo $RPMDIR`" $setnocheck openvswitch.spec cp $RPMDIR/RPMS/x86_64/*.rpm $HOME diff --git a/build/functions.sh b/build/functions.sh new file mode 100755 index 0000000..76a0683 --- /dev/null +++ b/build/functions.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright (c) 2016 Open Platform for NFV Project, Inc. and its contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo $0 [-d] [-k] + echo -d -- Test with DPDK + echo -k -- Load linux kernel module +} + +function delrpm() { + set +e + rpm -q $1 + if [ $? -eq 0 ]; then + sudo rpm -e --allmatches $1 + fi + set -e +} +function cleanrpms() { + delrpm openvswitch + delrpm dpdk-devel + delrpm dpdk-tools + delrpm dpdk-examples + delrpm dpdk +} +function uninstallrpms() { + cleanrpms +} diff --git a/build/instack_ovs.sh b/build/instack_ovs.sh deleted file mode 100755 index 4c258a4..0000000 --- a/build/instack_ovs.sh +++ /dev/null @@ -1,401 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Red Hat Inc. and others. -# therbert@redhat.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -set -e -declare -i CNT - -echo "===============================" -echo executing $0 $@ -echo path is $PATH - -usage() { - echo run BuildAndTestOVS -h for help -} - -while getopts "a:dg:hi:p:tu:v" opt; do - case "$opt" in - a) - kernel_major=${OPTARG} - ;; - d) - DPDK="yes" - setdpdk="-d" - ;; - g) - TAG=${OPTARG} - ;; - h|\?) - usage - exit 1 - ;; - i) - kernel_minor=${OPTARG} - ;; - p) - OVS_PATCH=${OPTARG} - ;; - t) - TESTRPM="yes" - ;; - u) - OVS_REPO_URL=${OPTARG} - ;; - v) - verbose="yes" - ;; - esac -done -# -# Default Config options -# -echo =============================================== -echo Default Configuration Options. -echo =============================================== -echo option NOCHECK is set to $NOCHECK -echo build DPDK option is set to $DPDK -echo DPDK Patch URL is set to $DPDK_PATCH -echo DPDK Version is set to $DPDK_VERSION -echo Option for OVS Kernel Module is set to $KMOD -echo =============================================== -if [[ $NOCHECK =~ "yes" ]]; then - setnocheck="-c" -fi -if [[ $KMOD =~ "yes" ]]; then - setkmod="-k" -fi - - - -if [ -z $OVS_REPO_URL ]; then - OVS_REPO_URL=https://github.com/openvswitch/ovs.git -fi - -if [ ! -z $kernel_major ] && [ ! -z $kernel_minor ]; then - kernel_version=$kernel_major.$kernel_minor - echo =================== - echo Will install kernel version: major is $kernel_major and minor is $kernel_minor - echo =================== -else - echo Will use default kernel in ovs test vm -fi - -if [ -z ${WORKSPACE+1} ]; then - # We are not being run by Jenkins. - export WORKSPACE=$HOME/opnfv/ovsnfv - mkdir -p opnfv - cd opnfv - git clone https://git.opnfv.org/ovsnfv -fi - -export BUILD_BASE=$WORKSPACE/build - -if [ ! -d $BUILD_BASE ] -then - mkdir -p $BUILD_BASE -fi - -if [ ! -f $BUILD_BASE/config ]; then - touch $BUILD_BASE/config -fi - -export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE -source $BUILD_BASE/config - -cd $BUILD_BASE -export TOPDIR=$BUILD_BASE - -export TMP_RELEASE_DIR=$TOPDIR/release -if [ ! -d $TMP_RELEASE_DIR ]; then - mkdir -p $TMP_RELEASE_DIR -fi - -export CACHE_DIR=$TOPDIR/cache -if [ ! -d $CACHE_DIR ]; then - mkdir -p $CACHE_DIR -fi -export TMPDIR=$TOPDIR/scratch -if [ ! -d $SCRATCH_DIR ]; then - mkdir -p $SCRATCH_DIR -fi - -rdo_images_uri=https://ci.centos.org/artifacts/rdo/images/liberty/delorean/stable - -vm_index=4 -RDO_RELEASE=liberty -SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null) -OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network" - -# check for dependancy packages -for i in rpm-build createrepo libguestfs-tools python-docutils bsdtar; do - if ! rpm -q $i > /dev/null; then - sudo yum install -y $i - fi -done - -# RDO Manager expects a stack user to exist, this checks for one -# and creates it if you are root -if ! id stack > /dev/null; then - sudo useradd stack; - sudo echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack - sudo echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack - sudo chmod 0440 /etc/sudoers.d/stack - echo 'Added user stack' -fi - -# ensure that I can ssh as the stack user -if ! sudo grep "$(cat ~/.ssh/id_rsa.pub)" /home/stack/.ssh/authorized_keys; then - if ! sudo ls -d /home/stack/.ssh/ ; then - sudo mkdir /home/stack/.ssh - sudo chown stack:stack /home/stack/.ssh - sudo chmod 700 /home/stack/.ssh - fi - USER=$(whoami) sudo sh -c "cat ~$USER/.ssh/id_rsa.pub >> /home/stack/.ssh/authorized_keys" - sudo chown stack:stack /home/stack/.ssh/authorized_keys -fi - -# clean up stack user previously build instack disk images -ssh -T ${SSH_OPTIONS[@]} stack@localhost "rm -f instack*.qcow2" - -# Yum repo setup for building the undercloud -if ! rpm -q rdo-release > /dev/null && [ "$1" != "-master" ]; then - sudo yum -y install yum-plugin-priorities - sudo yum-config-manager --disable openstack-${RDO_RELEASE} - sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo - sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo - sudo rm -f /etc/yum.repos.d/delorean-current.repo -elif [ "$1" == "-master" ]; then - sudo yum -y install yum-plugin-priorities - sudo yum-config-manager --disable openstack-${RDO_RELEASE} - sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo - sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo - sudo rm -f /etc/yum.repos.d/delorean-current.repo -fi - -# ensure the undercloud package is installed so we can build the undercloud -if ! rpm -q instack-undercloud > /dev/null; then - sudo yum install -y python-tripleoclient -fi - -# ensure openvswitch is installed -if ! rpm -q openvswitch > /dev/null; then - sudo yum install -y openvswitch -fi - -# ensure libvirt is installed -if ! rpm -q libvirt-daemon-kvm > /dev/null; then - sudo yum install -y libvirt-daemon-kvm -fi - -# clean this up incase it's there -sudo rm -f /tmp/instack.answers - -# ensure that no previous undercloud VMs are running -sudo ../ci/clean.sh -# and rebuild the bare undercloud VMs -ssh -T ${SSH_OPTIONS[@]} stack@localhost < /dev/null && [ $CNT -gt 0 ]; do - echo -n "." - sleep 3 - CNT=CNT-1 -done -# TODO fail if CNT=0 - -# yum update undercloud and reboot. -ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" < /dev/null; then - yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - fi - - yum -y install yum-plugin-priorities - curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo - curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo - - cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys - chown stack:stack /home/stack/.ssh/authorized_keys -EOI -# -# If using special kernel version, install on undercloud vm. -# -if [ ! -z $kernel_version ]; then - echo "Install special kernel version $kernel_version on undercloud" - ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" < /dev/null || echo -n '' -virsh undefine instack --remove-all-storage 2> /dev/null || echo -n '' -virsh vol-delete instack.qcow2 --pool default 2> /dev/null -rm -f /var/lib/libvirt/images/instack.qcow2 2> /dev/null +# Copyright (c) 2016 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# Clean off baremetal VMs in case they exist -for i in $(seq 0 $vm_index); do - virsh destroy baremetalbrbm_brbm1_$i 2> /dev/null || echo -n '' - virsh undefine baremetalbrbm_brbm1_$i --remove-all-storage 2> /dev/null || echo -n '' - virsh vol-delete baremetalbrbm_brbm1_${i}.qcow2 --pool default 2> /dev/null - rm -f /var/lib/libvirt/images/baremetalbrbm_brbm1_${i}.qcow2 2> /dev/null -done +set -e -# Clean off brbm bridges -virsh net-destroy brbm 2> /dev/null -virsh net-undefine brbm 2> /dev/null -vs-vsctl del-br brbm 2> /dev/null +echo "===============================" +echo "Requires sudo privileges" +echo executing $0 $@ +echo executing on machine `uname -a` -virsh net-destroy brbm1 2> /dev/null -virsh net-undefine brbm1 2> /dev/null -vs-vsctl del-br brbm1 2> /dev/null +if [ -z ${WORKSPACE+1} ]; then + # We are not being run by Jenkins. + export WORKSPACE=`pwd` +fi -# clean pub keys from root's auth keys -sed -i '/stack@instack.localdomain/d' /root/.ssh/authorized_keys -sed -i '/virtual-power-key/d' /root/.ssh/authorized_keys +export BUILD_BASE=$WORKSPACE +source $BUILD_BASE/../build/functions.sh -echo "Cleanup Completed" +cleanrpms +set +e +rm -rf ovsrpm +rm -rf rpmbuild +rm -rf rpms +rm -rf build +rm *.rpm +ccache -C +set -e +exit 0 -- cgit 1.2.3-korg