RPM: Apex: Refactor build scripts:

Remove reference to undercloud.
Fix dependencies list.
Refactor build ovs and build dpdk and test scripts.
Correct default dpdk version in build ovs script.
Remove references to instack VM -- no longer necessary.
Remove script using custom Apex-lite-liberty for testing
Consolidated functions and cleanup in response to reviewers.

Change-Id: I0875acd87580769c7cd7e87b97bb3e04f0159b32
Signed-off-by: Thomas F Herbert <therbert@redhat.com>

7 files changed, 193 insertions, 530 deletions

diff --git a/build/BuildAndTestOVS.sh b/build/BuildAndTestOVS.sh
index 10a2f05..f12ab3e 100755
--- a/build/BuildAndTestOVS.sh
+++ b/build/BuildAndTestOVS.sh
@@ -72,8 +72,26 @@ while getopts "a:dg:hi:p:tu:v" opt; do
     esac
 done
 
-if [ -z $TAG ]; then
-    TAG=master
+#
+# Default Config options
+#
+echo ===============================================
+echo Default Configuration Options.
+echo ===============================================
+echo option NOCHECK is set to $NOCHECK
+echo build DPDK option is set to $DPDK
+echo DPDK Patch URL is set to $DPDK_PATCH
+echo DPDK Version is set to $DPDK_VERSION
+echo Option for OVS Kernel Module is set to $KMOD
+echo ===============================================
+if [[ $NOCHECK =~ "yes" ]]; then
+    setnocheck="-c"
+fi
+if [[ $KMOD =~ "yes" ]]; then
+    setkmod="-k"
+fi
+if [[ $DPDK =~ "yes" ]]; then
+    setdpdk="-d"
 fi
 
 if [ -z $OVS_REPO_URL ]; then
@@ -89,33 +107,24 @@ else
     echo Will use default kernel in ovs test vm
 fi
 
+if [ -z $TAG ]; then
+    TAG=master
+fi
+
 if [ ! -z $DPDK ]; then
     setbuilddpdk="-d"
 fi
 
 if [ -z ${WORKSPACE+1} ]; then
     # We are not being run by Jenkins.
-    export WORKSPACE=$HOME/opnfv/ovsnfv
-    mkdir -p opnfv
-    cd opnfv
-    git clone https://git.opnfv.org/ovsnfv
-fi
-
-export BUILD_BASE=$WORKSPACE/build
-
-
-
-if [ ! -d $BUILD_BASE ]
-then
-    mkdir -p $BUILD_BASE
+    export WORKSPACE=`pwd`
 fi
 
-if [ ! -f $BUILD_BASE/config ]; then
-    touch $BUILD_BASE/config
+if [ -z ${BUILD_BASE+1} ]; then
+    export BUILD_BASE=$WORKSPACE
 fi
 
-export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE
-source $BUILD_BASE/config
+export PATH=$PATH:$BUILD_BASE
 
 cd $BUILD_BASE
 export TOPDIR=$BUILD_BASE
@@ -132,19 +141,35 @@ mkdir -p $RPMDIR/RPMS
 mkdir -p $RPMDIR/SOURCES
 mkdir -p $RPMDIR/SPECS
 mkdir -p $RPMDIR/SRPMS
-
-
+#
+# build dpdk rpm locally.
+#
+if [[ "$DPDK" =~ "yes" ]]; then
+    echo "==============================="
+    echo Build DPDK RPMs
+    echo
+    $BUILD_BASE/build_dpdk_rpm.sh -g $DPDK_VERSION
+fi
+#
+# Build locally and copy RPMS
+#
+echo "==============================="
+echo build OVS rpm locally
+echo
+    ./build_ovs_rpm.sh $setnocheck -g $TAG $setdpdk $setkmod -p $OVS_PATCH -u $OVS_REPO_URL
+#
+# Test rpm
+#
 if [ ! -z $TESTRPM ]; then
-    # Spawn VM to do the testing.
-    if [ ! -z $kernel_version ]; then
-        instack_ovs.sh -a $kernel_major -g $TAG -i $kernel_minor -p $OVS_PATCH -t -u $OVS_REPO_URL
-    else
-        instack_ovs.sh $setbuilddpdk -g $TAG -p $OVS_PATCH -t -u $OVS_REPO_URL
-    fi
-else
-    # Run build locally.
-    build_ovs_rpm.sh $setbuilddpdk -g $TAG -p $OVS_PATCH -u $OVS_REPO_URL
-    cp $HOME/rpmbuild/RPMS/* $TMP_RELEASE_DIR
+    ./test_ovs_rpm.sh $setdpdk $setkmod
 fi
 
+#
+# If tests pass, copy rpms to release dir
+#
+echo "==============================="
+echo copy rpms to release dir
+echo
+cp $RPMDIR/RPMS/x86_64/* $TMP_RELEASE_DIR
+
 exit 0
diff --git a/build/build_dpdk_rpm.sh b/build/build_dpdk_rpm.sh
index 1812dc3..91a7fdf 100755
--- a/build/build_dpdk_rpm.sh
+++ b/build/build_dpdk_rpm.sh
@@ -61,13 +61,14 @@ then
     rm -rf $TMPDIR
 fi
 
-echo "---------------------"
-echo "Install dependencies for dpdk"
-echo
-sudo yum -y install gcc make python-devel openssl-devel autoconf automake rpm-build \
-            redhat-rpm-config libtool libpcap-devel numactl-devel python-sphinx \
-            libvirt-devel
-
+function install_pre_reqs() {
+    echo "----------------------------------------"
+    echo Install dependencies for dpdk.
+    echo
+    sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \
+                kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \
+                libtool python-twisted-core desktop-file-utils groff PyQt4
+}
 
 mkdir -p $TMPDIR
 
diff --git a/build/build_ovs_rpm.sh b/build/build_ovs_rpm.sh
index b03f4ab..e28c23f 100755
--- a/build/build_ovs_rpm.sh
+++ b/build/build_ovs_rpm.sh
@@ -16,27 +16,11 @@
 set -e
 
 echo "==============================="
-echo executing $0 $@
-echo executing on machine `uname -a`
+echo "executing $0 $@"
+echo "executing on machine `uname -a`"
 
 usage() {
-    echo run BuildAndTestOVS -h for help
-}
-
-function delrpm() {
-    set +e
-    rpm -q $1
-    if [ $? -eq 0 ]; then
-        sudo rpm -e --allmatches $1
-    fi
-    set -e
-}
-function cleanrpms() {
-    delrpm openvswitch
-    delrpm dpdk-devel
-    delrpm dpdk-tools
-    delrpm dpdk-examples
-    delrpm dpdk
+    echo "run BuildAndTestOVS -h for help"
 }
 
 while getopts "cdg:hkp:u:v" opt; do
@@ -73,20 +57,26 @@ HOME=`pwd`
 TOPDIR=$HOME
 TMPDIR=$TOPDIR/ovsrpm
 
+BUILDDIR=$HOME
+
+source $BUILDDIR/functions.sh
+
 echo "---------------------------------------"
-echo Clean out old working dir
+echo "Clean out old working dir."
 echo
 if [ -d $TMPDIR ]
 then
     rm -rf $TMPDIR
 fi
 
-echo "----------------------------------------"
-echo Install pre-reqs.
-echo
-sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \
-       kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \
-       libtool python-twisted-core desktop-file-utils groff PyQt4
+function install_pre_reqs() {
+    echo "----------------------------------------"
+    echo "Install pre-reqs."
+    echo
+    sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \
+                kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \
+                libtool python-twisted-core desktop-file-utils groff PyQt4
+}
 
 VERSION=2.3.90
 os_type=fedora
@@ -95,7 +85,7 @@ kernel_version=$(uname -a | awk '{print $3}')
 RPMDIR=$HOME/rpmbuild
 
 echo "---------------------------------------"
-echo Clean out old reminents of old rpms and rpm _topdir.
+echo "Clean out old reminents of old rpms and rpm _topdir."
 echo
 
 rm openvswitch*.rpm || true
@@ -104,7 +94,7 @@ if [  -d $RPMDIR ]; then
 fi
 
 echo "---------------------------------------"
-echo Create new rpm _topdir.
+echo "Create new rpm _topdir."
 echo
 mkdir -p $HOME/rpmbuild/RPMS
 mkdir -p $HOME/rpmbuild/SOURCES
@@ -123,14 +113,15 @@ if [ ! -z $DPDK ]; then
     echo "----------------------------------"
     echo "Clone Fedora copr repo and copy files."
     echo
-    git clone https://github.com/tfherbert/ovs-snap.git 
+    git clone https://github.com/tfherbert/ovs-snap.git
     cd ovs-snap
     git checkout $COPR_OVS_VERSION
+    echo "-----------------------------------"
     cp $TMPDIR/ovs-snap/openvswitch.spec $RPMDIR/SPECS
     cp $TMPDIR/ovs-snap/* $RPMDIR/SOURCES
     snapgit=`grep "define snapver" $TMPDIR/ovs-snap/openvswitch.spec | cut -c26-33`
-    echo "-------------------------------------------"
-    echo "Remove old dpdk, ovs and dpdk development rpms"
+    echo "-----------------------------------------------"
+    echo "Remove any old installed ovs and dpdk rpms."
     echo
     cleanrpms
 
@@ -165,6 +156,7 @@ if [ ! -z $DPDK ]; then
     echo "--------------------------------------------"
     echo "Build openvswitch RPM"
     echo
+    rpmbuild -bb --define "_topdir `echo $RPMDIR`" $setnocheck openvswitch.spec
 else
     echo "-------------------------------------------------"
     echo "Build OVS without DPDK:"
@@ -181,7 +173,7 @@ else
         export VERSION=$basever
 
         echo "--------------------------------------------"
-        echo making distribution tarball for Open vswitch version $VERSION
+        echo "Making distribution tarball for Open vswitch version $VERSION"
         echo
         ./boot.sh
         ./configure
@@ -197,17 +189,27 @@ else
         curl --silent --output $HOME/rpmbuild/SOURCES/openvswitch-${VERSION}.tar.gz http://openvswitch.org/releases/openvswitch-${VERSION}.tar.gz
     fi
 
-    if [ ! -z $kmod ]; then
-        echo "--------------------------------------------"
-        echo "Building openvswitch kernel module RPM"
-        echo
-        rpmbuild -bb -D "kversion $kernel_version" -D "kflavors default" --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch-kmod-${os_type}.spec
-    fi
+fi
+#
+# This section is for building OVS kernel module.
+#
+if [ ! -z $kmod ]; then
+    cd $TMPDIR/ovs
     echo "--------------------------------------------"
-    echo "Build openvswitch RPM"
+    echo "Making distribution tarball for Open vswitch version $VERSION"
+    echo
+    ./boot.sh
+    ./configure
+    make dist
+    echo "--------------------------------------------"
+    echo "Copy distribution tarball to $HOME/rpmbuild/SOURCES"
+    echo
+    cp openvswitch-*.tar.gz $HOME/rpmbuild/SOURCES
+    echo "--------------------------------------------"
+    echo "Building openvswitch kernel module RPM"
     echo
+    rpmbuild -bb -D "kversion $kernel_version" -D "kflavors default" --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch-kmod-${os_type}.spec
 fi
-rpmbuild -bb --define "_topdir `echo $RPMDIR`" $setnocheck openvswitch.spec
 
 cp $RPMDIR/RPMS/x86_64/*.rpm $HOME
 
diff --git a/build/functions.sh b/build/functions.sh
new file mode 100755
index 0000000..76a0683
--- /dev/null
+++ b/build/functions.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright (c) 2016 Open Platform for NFV Project, Inc. and its contributors
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+set -e
+
+usage() {
+    echo $0 [-d] [-k]
+    echo -d -- Test with DPDK
+    echo -k -- Load linux kernel module
+}
+
+function delrpm() {
+    set +e
+    rpm -q $1
+    if [ $? -eq 0 ]; then
+        sudo rpm -e --allmatches $1
+    fi
+    set -e
+}
+function cleanrpms() {
+    delrpm openvswitch
+    delrpm dpdk-devel
+    delrpm dpdk-tools
+    delrpm dpdk-examples
+    delrpm dpdk
+}
+function uninstallrpms() {
+    cleanrpms
+}
diff --git a/build/instack_ovs.sh b/build/instack_ovs.sh
deleted file mode 100755
index 4c258a4..0000000
--- a/build/instack_ovs.sh
+++ /dev/null
@@ -1,401 +0,0 @@
-#!/bin/bash
-##############################################################################
-# Copyright (c) 2015 Red Hat Inc. and others.
-# therbert@redhat.com
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-set -e
-declare -i CNT
-
-echo "==============================="
-echo executing $0 $@
-echo path is $PATH
-
-usage() {
-    echo run BuildAndTestOVS -h for help
-}
-
-while getopts "a:dg:hi:p:tu:v" opt; do
-    case "$opt" in
-        a)
-            kernel_major=${OPTARG}
-            ;;
-        d)
-            DPDK="yes"
-            setdpdk="-d"
-            ;;
-        g)
-            TAG=${OPTARG}
-            ;;
-        h|\?)
-            usage
-            exit 1
-            ;;
-        i)
-            kernel_minor=${OPTARG}
-            ;;
-        p)
-            OVS_PATCH=${OPTARG}
-            ;;
-        t)
-            TESTRPM="yes"
-            ;;
-        u)
-            OVS_REPO_URL=${OPTARG}
-            ;;
-        v)
-            verbose="yes"
-            ;;
-    esac
-done
-#
-# Default Config options
-#
-echo ===============================================
-echo Default Configuration Options.
-echo ===============================================
-echo option NOCHECK is set to $NOCHECK
-echo build DPDK option is set to $DPDK
-echo DPDK Patch URL is set to $DPDK_PATCH
-echo DPDK Version is set to $DPDK_VERSION
-echo Option for OVS Kernel Module is set to $KMOD
-echo ===============================================
-if [[ $NOCHECK =~ "yes" ]]; then
-    setnocheck="-c"
-fi
-if [[ $KMOD =~ "yes" ]]; then
-    setkmod="-k"
-fi
-
-
-
-if [ -z $OVS_REPO_URL ]; then
-    OVS_REPO_URL=https://github.com/openvswitch/ovs.git
-fi
-
-if [ ! -z $kernel_major ] && [ ! -z $kernel_minor ]; then
-    kernel_version=$kernel_major.$kernel_minor
-    echo ===================
-    echo Will install kernel version: major is $kernel_major and minor is $kernel_minor
-    echo ===================
-else
-    echo Will use default kernel in ovs test vm
-fi
-
-if [ -z ${WORKSPACE+1} ]; then
-    # We are not being run by Jenkins.
-    export WORKSPACE=$HOME/opnfv/ovsnfv
-    mkdir -p opnfv
-    cd opnfv
-    git clone https://git.opnfv.org/ovsnfv
-fi
-
-export BUILD_BASE=$WORKSPACE/build
-
-if [ ! -d $BUILD_BASE ]
-then
-    mkdir -p $BUILD_BASE
-fi
-
-if [ ! -f $BUILD_BASE/config ]; then
-    touch $BUILD_BASE/config
-fi
-
-export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE
-source $BUILD_BASE/config
-
-cd $BUILD_BASE
-export TOPDIR=$BUILD_BASE
-
-export TMP_RELEASE_DIR=$TOPDIR/release
-if [ ! -d $TMP_RELEASE_DIR ]; then
-    mkdir -p $TMP_RELEASE_DIR
-fi
-
-export CACHE_DIR=$TOPDIR/cache
-if [ ! -d $CACHE_DIR ]; then
-    mkdir -p $CACHE_DIR
-fi
-export TMPDIR=$TOPDIR/scratch
-if [ ! -d $SCRATCH_DIR ]; then
-    mkdir -p $SCRATCH_DIR
-fi
-
-rdo_images_uri=https://ci.centos.org/artifacts/rdo/images/liberty/delorean/stable
-
-vm_index=4
-RDO_RELEASE=liberty
-SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null)
-OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network"
-
-# check for dependancy packages
-for i in rpm-build createrepo libguestfs-tools python-docutils bsdtar; do
-    if ! rpm -q $i > /dev/null; then
-        sudo yum install -y $i
-    fi
-done
-
-# RDO Manager expects a stack user to exist, this checks for one
-# and creates it if you are root
-if ! id stack > /dev/null; then
-    sudo useradd stack;
-    sudo echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack
-    sudo echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack
-    sudo chmod 0440 /etc/sudoers.d/stack
-    echo 'Added user stack'
-fi
-
-# ensure that I can ssh as the stack user
-if ! sudo grep "$(cat ~/.ssh/id_rsa.pub)" /home/stack/.ssh/authorized_keys; then
-    if ! sudo ls -d /home/stack/.ssh/ ; then
-        sudo mkdir /home/stack/.ssh
-        sudo chown stack:stack /home/stack/.ssh
-        sudo chmod 700 /home/stack/.ssh
-    fi
-    USER=$(whoami) sudo sh -c "cat ~$USER/.ssh/id_rsa.pub >> /home/stack/.ssh/authorized_keys"
-    sudo chown stack:stack /home/stack/.ssh/authorized_keys
-fi
-
-# clean up stack user previously build instack disk images
-ssh -T ${SSH_OPTIONS[@]} stack@localhost "rm -f instack*.qcow2"
-
-# Yum repo setup for building the undercloud
-if ! rpm -q rdo-release > /dev/null && [ "$1" != "-master" ]; then
-    sudo yum -y install yum-plugin-priorities
-    sudo yum-config-manager --disable openstack-${RDO_RELEASE}
-    sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo
-    sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
-    sudo rm -f /etc/yum.repos.d/delorean-current.repo
-elif [ "$1" == "-master" ]; then
-    sudo yum -y install yum-plugin-priorities
-    sudo yum-config-manager --disable openstack-${RDO_RELEASE}
-    sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo
-    sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
-    sudo rm -f /etc/yum.repos.d/delorean-current.repo
-fi
-
-# ensure the undercloud package is installed so we can build the undercloud
-if ! rpm -q instack-undercloud > /dev/null; then
-    sudo yum install -y python-tripleoclient
-fi
-
-# ensure openvswitch is installed
-if ! rpm -q openvswitch > /dev/null; then
-    sudo yum install -y openvswitch
-fi
-
-# ensure libvirt is installed
-if ! rpm -q libvirt-daemon-kvm > /dev/null; then
-    sudo yum install -y libvirt-daemon-kvm
-fi
-
-# clean this up incase it's there
-sudo rm -f /tmp/instack.answers
-
-# ensure that no previous undercloud VMs are running
-sudo ../ci/clean.sh
-# and rebuild the bare undercloud VMs
-ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
-    set -e
-    NODE_COUNT=5 NODE_CPU=2 NODE_MEM=8192 TESTENV_ARGS="--baremetal-bridge-names 'brbm brbm1 brbm2 brbm3'" instack-virt-setup
-EOI
-
-# let dhcp happen so we can get the ip
-# just wait instead of checking until we see an address
-# because there may be a previous lease that needs
-# to be cleaned up
-sleep 5
-
-# get the undercloud ip address
-UNDERCLOUD=$(grep instack /var/lib/libvirt/dnsmasq/default.leases | awk '{print $3}' | head -n 1)
-if [ -z "$UNDERCLOUD" ]; then
-  #if not found then dnsmasq may be using leasefile-ro
-  instack_mac=$(ssh -T ${SSH_OPTIONS[@]} stack@localhost "virsh domiflist instack" | grep default | \
-                grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+")
-  UNDERCLOUD=$(arp -e | grep ${instack_mac} | awk {'print $1'})
-
-  if [ -z "$UNDERCLOUD" ]; then
-    echo "\n\nNever got IP for Instack. Can Not Continue."
-    exit 1
-  fi
-else
-   echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
-fi
-
-# ensure that we can ssh to the undercloud
-CNT=10
-while ! ssh -T ${SSH_OPTIONS[@]}  "root@$UNDERCLOUD" "echo ''" > /dev/null && [ $CNT -gt 0 ]; do
-    echo -n "."
-    sleep 3
-    CNT=CNT-1
-done
-# TODO fail if CNT=0
-
-# yum update undercloud and reboot.
-ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
-    set -e
-
-    echo "----------------------------------------------------------------"
-    echo yum update and install pciutils prereqs for DPDK tools and samples.
-    echo
-    yum -y update
-    yum -y install pciutils libvirt
-EOI
-
-# reboot VM
-ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
-    virsh reboot instack
-EOI
-sleep 30
-
-# yum repo, triple-o package and ssh key setup for the undercloud
-echo "Install epel-release on undercloud"
-ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
-    set -e
-
-    if ! rpm -q epel-release > /dev/null; then
-        yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    fi
-
-    yum -y install yum-plugin-priorities
-    curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo
-    curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
-
-    cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys
-    chown stack:stack /home/stack/.ssh/authorized_keys
-EOI
-#
-# If using special kernel version, install on undercloud vm.
-#
-if [ ! -z $kernel_version ]; then
-    echo "Install special kernel version $kernel_version on undercloud"
-    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
-    set -e
-    yum -y install gcc ncurses ncurses-devel bc xz rpm-build
-    echo wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
-    wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
-    echo wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
-    wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
-    echo rpm -i kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
-    rpm -i kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
-    echo rpm -i kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
-    rpm -i kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
-
-    echo cd /lib/modules/$kernel_version-1.el6.elrepo.x86_64
-    cd /lib/modules/$kernel_version-1.el6.elrepo.x86_64
-    echo rm -f build
-    rm -f build
-    echo ln -s /usr/src/kernels/$kernel_version-1.el6.elrepo.x86_64 build
-    ln -s /usr/src/kernels/$kernel_version-1.el6.elrepo.x86_64 build
-    #echo rm -f source
-    #rm -f source
-    #echo ln -s ./build source
-    #ln -s ./build source
-EOI
-else
-    #
-    # Install latest stable kernel.
-    #
-    echo "Install devel-kernel and elrepo on undercloud"
-    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
-        echo Install latest stable kernel
-        set -e
-        yum install -y kernel kernel-devel
-        rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
-        rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
-EOI
-fi
-
-# copy instackenv file for future virt deployments
-echo copy instackenv file for future virt deployments
-if [ ! -d stack ]; then mkdir stack; fi
-scp ${SSH_OPTIONS[@]} stack@$UNDERCLOUD:instackenv.json stack/instackenv.json
-
-
-#
-# If using special kernel version, reboot undercloud vm
-#
-echo If using special kernel version, reboot undercloud vm
-if [ -z $kernel_version ]; then
-    ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
-        virsh reboot instack
-EOI
-    sleep 15
-fi
-
-#
-# Copy build and test scripts to undercloud vm.
-# If special kernel is required, build rpm on undercloud vm otherwise build
-# it locally.
-#
-echo Copy build and test scripts to undercloud vm.
-echo BUILD_BASE is $BUILD_BASE
-scp ${SSH_OPTIONS[@]} $BUILD_BASE/build_ovs_rpm.sh stack@$UNDERCLOUD:
-scp ${SSH_OPTIONS[@]} $BUILD_BASE/test_ovs_rpm.sh stack@$UNDERCLOUD:
-#
-# build dpdk rpm locally.
-#
-if [[ "$DPDK" =~ "yes" ]]; then
-    echo Build DPDK RPMs
-    ./build_dpdk_rpm.sh -g $DPDK_VERSION
-fi
-#
-# Build rpm on undercloud if custom kernel module is required otherwise build
-# locally.
-#
-if [ ! -z $kernel_version ]; then
-    echo build rpm on undercloud with kernel version $kernel_version
-    ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
-        ./build_ovs_rpm.sh -a $kernel_major $setnocheck -g $TAG -i $kernel_minor -k -p $OVS_PATCH -u $OVS_REPO_URL
-EOI
-    scp ${SSH_OPTIONS[@]} stack@UNDERCLOUD:*.rpm $RPMDIR/RPMS/
-elif [[ "$DPDK" =~ "yes" ]]; then
-    echo Build ovs with DPDK locally
-    #
-    # Build locally and copy RPMS to undercloud vm for testing
-    # and copy RPMS to temporary release dir.
-    #
-    ./build_ovs_rpm.sh $setnocheck -d -g $TAG -p $OVS_PATCH -u $OVS_REPO_URL
-else
-    # Build locally and copy RPMS to undercloud vm for testing
-    # and copy RPMS to temporary release dir.
-    #
-    echo build OVS rpm locally
-    ./build_ovs_rpm.sh $setnocheck -g $TAG $setkmod -p $OVS_PATCH -u $OVS_REPO_URL
-fi
-#
-# Test rpm on undercloud vm
-# TODO: Undercloud VM doesn't support sse3 instruction needed set to run DPDK
-#
-if [ ! -z $TESTRPM ]; then
-    if [ -z $DPDK ]; then
-        echo "-----------------------------------------"
-        echo Test rpm on undercloud vm
-        echo Copy all RPMS to undercloud for testing.
-        echo
-        scp ${SSH_OPTIONS[@]} $RPMDIR/RPMS/x86_64/* stack@$UNDERCLOUD:
-        scp ${SSH_OPTIONS[@]} $RPMDIR/SOURCES/dpdk*.rpm stack@$UNDERCLOUD:
-        ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
-            ./test_ovs_rpm.sh $setdpdk $setkmod
-EOI
-    else
-        echo "-----------------------------------------"
-        echo "TODO: Undercloud VM doesn't support sse3 instruction needed DPDK."
-        echo "DPDK is required, therefore test DPDK/OVS RPM in host"
-        echo
-        ./test_ovs_rpm.sh $setdpdk $setkmod
-    fi
-fi
-
-#
-# If tests pass, copy rpms to release dir
-#
-echo copy rpms to release dir
-echo copy rpms from undercloud back to $TMP_RELEASE_DIR in host
-cp $RPMDIR/RPMS/x86_64/* $TMP_RELEASE_DIR
-
-exit 0
diff --git a/build/test_ovs_rpm.sh b/build/test_ovs_rpm.sh
index d0b9174..6fff26a 100755
--- a/build/test_ovs_rpm.sh
+++ b/build/test_ovs_rpm.sh
@@ -17,28 +17,19 @@
 set -e
 
 echo "==============================="
+echo "Requires sudo privileges"
 echo executing $0 $@
 echo executing on machine `uname -a`
 
 
 usage() {
-    echo run BuildAndTest -h for help
+    echo $0 [-d] [-k]
+    echo -d -- Test with DPDK
+    echo -k -- Load linux kernel module
 }
 
-function delrpm() {
-    set +e
-    rpm -q $1
-    if [ $? -eq 0 ]; then
-        sudo rpm -e --allmatches $1
-    fi
-    set -e
-}
-function cleanrpms() {
-    delrpm openvswitch
-    delrpm dpdk-devel
-    delrpm dpdk-tools
-    delrpm dpdk-examples
-    delrpm dpdk
+function uninstallrpms() {
+    cleanrpms
 }
 
 while getopts "dg:hkp:u:v" opt; do
@@ -69,6 +60,9 @@ HOME=`pwd`
 TOPDIR=$HOME
 TMPDIR=$TOPDIR/ovsrpm
 
+BUILDDIR=$HOME
+source $BUILDDIR/functions.sh
+
 if [ -d $TMPDIR ]
 then
     rm -rf $TMPDIR
@@ -134,5 +128,6 @@ sudo ovs-vsctl add-br brtest
 sudo ovs-ofctl dump-flows brtest
 sudo ovs-vsctl del-br brtest
 sudo service openvswitch stop
+uninstallrpms
 
 exit 0
diff --git a/ci/clean.sh b/ci/clean.sh
index b1033b0..367c61d 100755
--- a/ci/clean.sh
+++ b/ci/clean.sh
@@ -1,43 +1,42 @@
-#!/usr/bin/env bash
-##############################################################################
-# Copyright (c) 2016 Red Hat Inc. and others.
-# therbert@redhat.com
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-#Clean script to uninstall provisioning server for Apex
-#author: Dan Radez (dradez@redhat.com)
-#
-vm_index=4
+#/bin/bash
 
-# Clean off instack VM
-virsh destroy instack 2> /dev/null || echo -n ''
-virsh undefine instack --remove-all-storage 2> /dev/null || echo -n ''
-virsh vol-delete instack.qcow2 --pool default 2> /dev/null
-rm -f /var/lib/libvirt/images/instack.qcow2 2> /dev/null
+# Copyright (c) 2016 Red Hat Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
 
-# Clean off baremetal VMs in case they exist
-for i in $(seq 0 $vm_index); do
-  virsh destroy baremetalbrbm_brbm1_$i 2> /dev/null || echo -n ''
-  virsh undefine baremetalbrbm_brbm1_$i --remove-all-storage 2> /dev/null || echo -n ''
-  virsh vol-delete baremetalbrbm_brbm1_${i}.qcow2 --pool default 2> /dev/null
-  rm -f /var/lib/libvirt/images/baremetalbrbm_brbm1_${i}.qcow2 2> /dev/null
-done
+set -e
 
-# Clean off brbm bridges
-virsh net-destroy brbm 2> /dev/null
-virsh net-undefine brbm 2> /dev/null
-vs-vsctl del-br brbm 2> /dev/null
+echo "==============================="
+echo "Requires sudo privileges"
+echo executing $0 $@
+echo executing on machine `uname -a`
 
-virsh net-destroy brbm1 2> /dev/null
-virsh net-undefine brbm1 2> /dev/null
-vs-vsctl del-br brbm1 2> /dev/null
+if [ -z ${WORKSPACE+1} ]; then
+    # We are not being run by Jenkins.
+    export WORKSPACE=`pwd`
+fi
 
-# clean pub keys from root's auth keys
-sed -i '/stack@instack.localdomain/d' /root/.ssh/authorized_keys
-sed -i '/virtual-power-key/d' /root/.ssh/authorized_keys
+export BUILD_BASE=$WORKSPACE
+source $BUILD_BASE/../build/functions.sh
 
 
-echo "Cleanup Completed"
+cleanrpms
+set +e
+rm -rf ovsrpm
+rm -rf rpmbuild
+rm -rf rpms
+rm -rf build
+rm *.rpm
+ccache -C
+set -e
+exit 0