Introduce ability to accept command arguments to build with DPDK or Linux kernel.

Specify patches to apply, build kernel module RPM and build with a special
version of the kernel.

Also instack_ovs.sh creates a VM for building and initial testing of RPMs and
copies scripts to VMs

build_ovs_rpm.sh builds the RPM according to command line options.
test_ovs_rpm.sh tests the RPM by installing the RPM and running some ovs
commands to create a bridge and dump flows.

Also add two scripts generally to be executed in VM to do actual building and
Testing.

This 2nd series adds the following:

Use fedora spec file for kmod builds. Added Explanations to configuration options.
Fixes in response to review comments. Rename script for clarity and change
references: buildtestovs.sh -> BuildTestOVS.sh
Fix nocheck option, print default options, fix kmod option, fix test of kmod.

For command line options execute build/buildtestovs.sh -h

Change-Id: I415ca39afab27482b1cb473d392f48b36c8e0745
Signed-off-by: Thomas F Herbert <therbert@redhat.com>

7 files changed, 815 insertions, 89 deletions

diff --git a/build/BuildAndTestOVS.sh b/build/BuildAndTestOVS.sh
new file mode 100755
index 0000000..ff2495c
--- /dev/null
+++ b/build/BuildAndTestOVS.sh
@@ -0,0 +1,145 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc. and others.
+# therbert@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -e
+
+echo "==============================="
+echo executing $0 $@
+
+usage() {
+    echo "$0 -a <kernel major> -d -g <OVS TAG> -h\
+             -i <kernel minor> -p <patch url> -t -u <OVS URL> -v <verbose\
+    -a <kernel major> -- Specify major release if special kernel is required\
+    The default kernel is Centos 7.2 kernel after upgrade.\
+    -d <dpdk>         -- Specify dpdk build.\
+                    The default is to build ovs for linux kernel data path.\
+    -g <OVS TAG>      -- OVS release tag or branch to build such as 2.4.\
+                    The default is master.\
+    -h print this message\
+    -i <kernel minor> -- Specify minor release if special kernel is required.\
+                    The default kernel is Centos 7.2 kernel after upgrade.\
+    -p <patch url>    -- Specify url to patches if required for ovs rpm.\
+    -t                -- Test rpm.\
+    -u <OVS URL>      -- path to OVS repo if using fork for patch.\
+                    The default is https://github.com/openvswitch/ovs.git\
+    -v                -- Set verbose mode."
+}
+
+while getopts "a:dg:hi:p:tu:v" opt; do
+    case "$opt" in
+        a)
+            kernel_major=${OPTARG}
+            ;;
+        d)
+            DPDK="yes"
+            ;;
+        g)
+            TAG=${OPTARG}
+            ;;
+        h)
+            usage
+            exit 1
+            ;;
+        i)
+            kernel_minor=${OPTARG}
+            ;;
+        p)
+            OVS_PATCH=${OPTARG}
+            ;;
+        t)
+            TESTRPM="yes"
+            ;;
+        u)
+            OVS_REPO_URL=${OPTARG}
+            ;;
+        v)
+            verbose="yes"
+            ;;
+    esac
+done
+
+if [ -z $TAG ]; then
+    TAG=master
+fi
+
+if [ -z $OVS_REPO_URL ]; then
+    OVS_REPO_URL=https://github.com/openvswitch/ovs.git
+fi
+
+if [ ! -z $kernel_major ] && [ ! -z $kernel_minor ]; then
+    kernel_version=$kernel_major.$kernel_minor
+    echo ===================
+    echo Will install kernel version: major is $kernel_major and minor is $kernel_minor
+    echo ===================
+else
+    echo Will use default kernel in ovs test vm
+fi
+
+if [ -z ${WORKSPACE+1} ]; then
+    # We are not being run by Jenkins.
+    export WORKSPACE=$HOME/opnfv/ovsnfv
+    mkdir -p opnfv
+    cd opnfv
+    git clone https://git.opnfv.org/ovsnfv
+fi
+
+export BUILD_BASE=$WORKSPACE/build
+
+
+
+if [ ! -d $BUILD_BASE ]
+then
+    mkdir -p $BUILD_BASE
+fi
+
+if [ ! -f $BUILD_BASE/config ]; then
+    touch $BUILD_BASE/config
+fi
+
+export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE
+source $BUILD_BASE/config
+
+cd $BUILD_BASE
+export TOPDIR=$BUILD_BASE
+
+# build variables
+
+export TMP_RELEASE_DIR=$TOPDIR/release
+export CACHE_DIR=$TOPDIR/cache
+export TMPDIR=$TOPDIR/scratch
+export RPMDIR=$TOPDIR/rpmbuild
+
+
+mkdir -p $RPMDIR/RPMS
+mkdir -p $RPMDIR/SOURCES
+mkdir -p $RPMDIR/SPECS
+mkdir -p $RPMDIR/SRPMS
+
+
+if [ ! -z $TESTRPM ]; then
+    # Spawn VM to do the testing.
+    if [ ! -z $kernel_version ]; then
+        instack_ovs.sh -a $kernel_major -g $TAG -i $kernel_minor -p $OVS_PATCH -t -u $OVS_REPO_URL
+    else
+        instack_ovs.sh -g $TAG -p $OVS_PATCH -t -u $OVS_REPO_URL
+    fi
+else
+    # Run build locally.
+    build_ovs_rpm.sh -d -g -p $OVS_PATCH -u $OVS_REPO_URL
+    cp $HOME/rpmbuild/RPMS/* $TMP_RELEASE_DIR
+fi
+
+echo "--------------------------------------------------"
+echo "Build OVS RPM from upstream git $OVS_REPO_URL version $TAG"
+if [ ! -z $OVS_PATCH ]; then
+    echo "Apply patches from: $OVS_PATCH"
+fi
+echo
+
+exit 0
diff --git a/build/build_ovs_rpm.sh b/build/build_ovs_rpm.sh
new file mode 100755
index 0000000..60b92e6
--- /dev/null
+++ b/build/build_ovs_rpm.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc. and others.
+# therbert@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -e
+declare -i CNT
+
+echo "==============================="
+echo executing $0 $@
+echo executing on machine `uname -a`
+
+usage() {
+    echo run BuildAndTestOVS -h for help
+}
+
+while getopts "cdg:hkp:u:v" opt; do
+    case "$opt" in
+        c)
+            setnocheck="--without check"
+            ;;
+        d)
+            DPDK="yes"
+            ;;
+        g)
+            TAG=${OPTARG}
+            ;;
+        h|\?)
+            usage
+            exit 1
+            ;;
+        k)
+            kmod="yes"
+            ;;
+        p)
+            OVS_PATCH=${OPTARG}
+            ;;
+        u)
+            OVS_REPO_URL=${OPTARG}
+            ;;
+        v)
+            verbose="yes"
+            ;;
+    esac
+done
+
+HOME=`pwd`
+TOPDIR=$HOME
+TMPDIR=$TOPDIR/ovsrpm
+
+if [ -d $TMPDIR ]
+then
+    rm -rf $TMPDIR
+fi
+
+sudo yum -y install gcc make python-devel openssl-devel kernel-devel graphviz \
+       kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \
+       libtool
+
+VERSION=2.3.90
+os_type=fedora
+kernel_version=$(uname -a | awk '{print $3}')
+
+mkdir -p $TMPDIR
+
+cd $TMPDIR
+
+mkdir -p $HOME/rpmbuild/RPMS
+mkdir -p $HOME/rpmbuild/SOURCES
+mkdir -p $HOME/rpmbuild/SPECS
+mkdir -p $HOME/rpmbuild/SRPMS
+
+RPMDIR=$HOME/rpmbuild
+
+
+echo "---------------------"
+echo "Clone git repo $OVS_REPO_URL and checkout branch or tag $TAG"
+echo
+git clone $OVS_REPO_URL
+
+cd ovs
+echo "--------------------"
+echo "Checkout OVS $TAG"
+echo
+if [[ ! "$TAG" =~ "master" ]]; then
+    git checkout $TAG
+fi
+if [[ ! "$OVS_PATCH" =~ "no" ]]; then
+    echo "Apply patches from $OVS_PATCH"
+fi
+./boot.sh
+if [ ! -z $DPDK ]; then
+    ./configure --with-dpdk
+else
+    ./configure --with-linux=/lib/modules/`uname -r`/build
+fi
+echo "--------------------"
+echo "Make OVS $TAG"
+echo
+make
+
+if [[ "$TAG" =~ "master" ]]; then
+    v=$($TMPDIR/ovs/utilities/ovs-vsctl --version | head -1 | cut -d' ' -f4)
+    export VERSION=$v
+else
+    export VERSION=${TAG:1}
+fi
+
+echo making RPM for Open vswitch version $VERSION
+make dist
+
+echo cp openvswitch-*.tar.gz $HOME/rpmbuild/SOURCES
+cp openvswitch-*.tar.gz $HOME/rpmbuild/SOURCES
+
+if [ ! -z $kmod ]; then
+    echo "Building kernel module..."
+    rpmbuild -bb -D "kversion $kernel_version" -D "kflavors default" --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch-kmod-${os_type}.spec
+echo " Kernel RPM built!"
+fi
+
+echo "Building User Space..."
+rpmbuild -bb --define "_topdir `echo $RPMDIR`" $setnocheck rhel/openvswitch.spec
+
+cp $RPMDIR/RPMS/x86_64/*.rpm $HOME
+
+exit 0
diff --git a/build/config b/build/config
index 337e07d..a197871 100644
--- a/build/config
+++ b/build/config
@@ -1,5 +1,26 @@
 # config file to be populated with configuration parameters for build.
 
-# OVS TAG
+# OVS configuration.
+#
+# OVS TAG or revision to build. The default is master which causes an RPM to be
+# built from the top of current master. The tag uses the current ovs naming and
+# release convention. This option can be overriden in the command line.
+#
 export OVSTAG=master
-#export NOCHECK=true
+#
+# when NOCHECK is yes, the ovs rpm is build with the --without check option which
+# Disables running "make check" as the RPM is built.
+#
+export NOCHECK=yes
+#
+# URL to external URL for patches to be applied for Open vSwitch.
+#
+export PATCH=none
+#
+# DPDK configuration.
+#
+export DPDK_PATCH=none
+#
+# When KMOD is set, the Linux kernel module is built and tested.
+#
+export KMOD=yes
diff --git a/build/instack_ovs.sh b/build/instack_ovs.sh
new file mode 100755
index 0000000..c4378d9
--- /dev/null
+++ b/build/instack_ovs.sh
@@ -0,0 +1,361 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2015 Red Hat Inc. and others.
+# therbert@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+declare -i CNT
+
+echo "==============================="
+echo executing $0 $@
+echo path is $PATH
+
+usage() {
+    echo run BuildAndTestOVS -h for help
+}
+
+while getopts "a:dg:hi:p:tu:v" opt; do
+    case "$opt" in
+        a)
+            kernel_major=${OPTARG}
+            ;;
+        d)
+            DPDK="yes"
+            ;;
+        g)
+            TAG=${OPTARG}
+            ;;
+        h|\?)
+            usage
+            exit 1
+            ;;
+        i)
+            kernel_minor=${OPTARG}
+            ;;
+        p)
+            OVS_PATCH=${OPTARG}
+            ;;
+        t)
+            TESTRPM="yes"
+            ;;
+        u)
+            OVS_REPO_URL=${OPTARG}
+            ;;
+        v)
+            verbose="yes"
+            ;;
+    esac
+done
+#
+# Default Config options
+#
+echo ===============================================
+echo Default Configuration Options.
+echo ===============================================
+echo option NOCHECK is set to $NOCHECK
+echo DPDK Patch URL is set to $DPDK_PATCH
+echo Build and Test OVS Kernel Module is set to $KMOD
+echo ===============================================
+if [[ $NOCHECK =~ "yes" ]]; then
+    setnocheck="-c"
+fi
+if [[ $KMOD =~ "yes" ]]; then
+    setkmod="-k"
+fi
+
+
+
+if [ -z $OVS_REPO_URL ]; then
+    OVS_REPO_URL=https://github.com/openvswitch/ovs.git
+fi
+
+if [ ! -z $kernel_major ] && [ ! -z $kernel_minor ]; then
+    kernel_version=$kernel_major.$kernel_minor
+    echo ===================
+    echo Will install kernel version: major is $kernel_major and minor is $kernel_minor
+    echo ===================
+else
+    echo Will use default kernel in ovs test vm
+fi
+
+if [ -z ${WORKSPACE+1} ]; then
+    # We are not being run by Jenkins.
+    export WORKSPACE=$HOME/opnfv/ovsnfv
+    mkdir -p opnfv
+    cd opnfv
+    git clone https://git.opnfv.org/ovsnfv
+fi
+
+export BUILD_BASE=$WORKSPACE/build
+
+if [ ! -d $BUILD_BASE ]
+then
+    mkdir -p $BUILD_BASE
+fi
+
+if [ ! -f $BUILD_BASE/config ]; then
+    touch $BUILD_BASE/config
+fi
+
+export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE
+source $BUILD_BASE/config
+
+cd $BUILD_BASE
+export TOPDIR=$BUILD_BASE
+
+export TMP_RELEASE_DIR=$TOPDIR/release
+if [ ! -d $TMP_RELEASE_DIR ]; then
+    mkdir -p $TMP_RELEASE_DIR
+fi
+
+export CACHE_DIR=$TOPDIR/cache
+if [ ! -d $CACHE_DIR ]; then
+    mkdir -p $CACHE_DIR
+fi
+export TMPDIR=$TOPDIR/scratch
+if [ ! -d $SCRATCH_DIR ]; then
+    mkdir -p $SCRATCH_DIR
+fi
+
+rdo_images_uri=https://ci.centos.org/artifacts/rdo/images/liberty/delorean/stable
+
+vm_index=4
+RDO_RELEASE=liberty
+SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null)
+OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network"
+
+# check for dependancy packages
+for i in rpm-build createrepo libguestfs-tools python-docutils bsdtar; do
+    if ! rpm -q $i > /dev/null; then
+        sudo yum install -y $i
+    fi
+done
+
+# RDO Manager expects a stack user to exist, this checks for one
+# and creates it if you are root
+if ! id stack > /dev/null; then
+    sudo useradd stack;
+    sudo echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack
+    sudo echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack
+    sudo chmod 0440 /etc/sudoers.d/stack
+    echo 'Added user stack'
+fi
+
+# ensure that I can ssh as the stack user
+if ! sudo grep "$(cat ~/.ssh/id_rsa.pub)" /home/stack/.ssh/authorized_keys; then
+    if ! sudo ls -d /home/stack/.ssh/ ; then
+        sudo mkdir /home/stack/.ssh
+        sudo chown stack:stack /home/stack/.ssh
+        sudo chmod 700 /home/stack/.ssh
+    fi
+    USER=$(whoami) sudo sh -c "cat ~$USER/.ssh/id_rsa.pub >> /home/stack/.ssh/authorized_keys"
+    sudo chown stack:stack /home/stack/.ssh/authorized_keys
+fi
+
+# clean up stack user previously build instack disk images
+ssh -T ${SSH_OPTIONS[@]} stack@localhost "rm -f instack*.qcow2"
+
+# Yum repo setup for building the undercloud
+if ! rpm -q rdo-release > /dev/null && [ "$1" != "-master" ]; then
+    sudo yum -y install yum-plugin-priorities
+    sudo yum-config-manager --disable openstack-${RDO_RELEASE}
+    sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo
+    sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
+    sudo rm -f /etc/yum.repos.d/delorean-current.repo
+elif [ "$1" == "-master" ]; then
+    sudo yum -y install yum-plugin-priorities
+    sudo yum-config-manager --disable openstack-${RDO_RELEASE}
+    sudo curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo
+    sudo curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
+    sudo rm -f /etc/yum.repos.d/delorean-current.repo
+fi
+
+# ensure the undercloud package is installed so we can build the undercloud
+if ! rpm -q instack-undercloud > /dev/null; then
+    sudo yum install -y python-tripleoclient
+fi
+
+# ensure openvswitch is installed
+if ! rpm -q openvswitch > /dev/null; then
+    sudo yum install -y openvswitch
+fi
+
+# ensure libvirt is installed
+if ! rpm -q libvirt-daemon-kvm > /dev/null; then
+    sudo yum install -y libvirt-daemon-kvm
+fi
+
+# clean this up incase it's there
+sudo rm -f /tmp/instack.answers
+
+# ensure that no previous undercloud VMs are running
+clean.sh
+# and rebuild the bare undercloud VMs
+ssh -T ${SSH_OPTIONS[@]} stack@localhost <<EOI
+    set -e
+    NODE_COUNT=5 NODE_CPU=2 NODE_MEM=8192 TESTENV_ARGS="--baremetal-bridge-names 'brbm brbm1 brbm2 brbm3'" instack-virt-setup
+EOI
+
+# let dhcp happen so we can get the ip
+# just wait instead of checking until we see an address
+# because there may be a previous lease that needs
+# to be cleaned up
+sleep 5
+
+# get the undercloud ip address
+UNDERCLOUD=$(grep instack /var/lib/libvirt/dnsmasq/default.leases | awk '{print $3}' | head -n 1)
+if [ -z "$UNDERCLOUD" ]; then
+  #if not found then dnsmasq may be using leasefile-ro
+  instack_mac=$(ssh -T ${SSH_OPTIONS[@]} stack@localhost "virsh domiflist instack" | grep default | \
+                grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+")
+  UNDERCLOUD=$(arp -e | grep ${instack_mac} | awk {'print $1'})
+
+  if [ -z "$UNDERCLOUD" ]; then
+    echo "\n\nNever got IP for Instack. Can Not Continue."
+    exit 1
+  fi
+else
+   echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
+fi
+
+# ensure that we can ssh to the undercloud
+CNT=10
+while ! ssh -T ${SSH_OPTIONS[@]}  "root@$UNDERCLOUD" "echo ''" > /dev/null && [ $CNT -gt 0 ]; do
+    echo -n "."
+    sleep 3
+    CNT=CNT-1
+done
+# TODO fail if CNT=0
+
+# yum update undercloud and reboot.
+ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
+set -e
+
+echo yum -y update
+yum -y update
+EOI
+
+virsh reboot instack
+sleep 30
+
+# yum repo, triple-o package and ssh key setup for the undercloud
+echo "Install epel-release on undercloud"
+ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
+    set -e
+
+    if ! rpm -q epel-release > /dev/null; then
+        yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    fi
+
+    yum -y install yum-plugin-priorities
+    curl -o /etc/yum.repos.d/delorean.repo http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/delorean.repo
+    curl -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos7-liberty/delorean-deps.repo
+
+    cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys
+    chown stack:stack /home/stack/.ssh/authorized_keys
+EOI
+#
+# If using special kernel version, install on undercloud vm.
+#
+if [ ! -z $kernel_version ]; then
+    echo "Install special kernel version $kernel_version on undercloud"
+    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
+    set -e
+    yum -y install gcc ncurses ncurses-devel bc xz rpm-build
+    echo wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
+    wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
+    echo wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
+    wget --quiet http://mirrors.neterra.net/elrepo/kernel/el6/x86_64/RPMS/kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
+    echo rpm -i kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
+    rpm -i kernel-ml-$kernel_version-1.el6.elrepo.x86_64.rpm
+    echo rpm -i kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
+    rpm -i kernel-ml-devel-$kernel_version-1.el6.elrepo.x86_64.rpm
+
+    echo cd /lib/modules/$kernel_version-1.el6.elrepo.x86_64
+    cd /lib/modules/$kernel_version-1.el6.elrepo.x86_64
+    echo rm -f build
+    rm -f build
+    echo ln -s /usr/src/kernels/$kernel_version-1.el6.elrepo.x86_64 build
+    ln -s /usr/src/kernels/$kernel_version-1.el6.elrepo.x86_64 build
+    #echo rm -f source
+    #rm -f source
+    #echo ln -s ./build source
+    #ln -s ./build source
+EOI
+else
+    #
+    # Install latest stable kernel.
+    #
+    echo "Install devel-kernel and elrepo on undercloud"
+    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" <<EOI
+        echo Install latest stable kernel
+        set -e
+        yum install -y kernel kernel-devel
+        rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
+        rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
+EOI
+fi
+
+# copy instackenv file for future virt deployments
+echo copy instackenv file for future virt deployments
+if [ ! -d stack ]; then mkdir stack; fi
+scp ${SSH_OPTIONS[@]} stack@$UNDERCLOUD:instackenv.json stack/instackenv.json
+
+
+#
+# If using special kernel version, reboot undercloud vm
+#
+echo If using special kernel version, reboot undercloud vm
+if [ -z $kernel_version ]; then
+    virsh reboot instack
+    sleep 15
+fi
+
+#
+# Copy build and test scripts to undercloud vm.
+# If special kernel is required, build rpm on undercloud vm otherwise build
+# it locally.
+#
+echo Copy build and test scripts to undercloud vm.
+echo BUILD_BASE is $BUILD_BASE
+scp ${SSH_OPTIONS[@]} $BUILD_BASE/build_ovs_rpm.sh stack@$UNDERCLOUD:
+scp ${SSH_OPTIONS[@]} $BUILD_BASE/test_ovs_rpm.sh stack@$UNDERCLOUD:
+
+#
+# build rpm on undercloud.
+#
+if [ ! -z $kernel_version ]; then
+    echo build rpm on undercloud with kernel version $kernel_version
+    ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
+        ./build_ovs_rpm.sh -a $kernel_major $setnocheck -g $TAG -i $kernel_minor -k -p $OVS_PATCH -u $OVS_REPO_URL
+EOI
+else
+    # build locally and copy RPMS to undercloud vm for testing
+    # and copy RPMS to temporary release dir.
+    #
+    echo build rpm on undercloud
+    ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
+        ./build_ovs_rpm.sh $setnocheck -g $TAG $setkmod -p $OVS_PATCH -u $OVS_REPO_URL
+EOI
+fi
+#
+# Test rpm on undercloud vm
+#
+if [[ ! -z $TESTRPM ]]; then
+    echo Test rpm on undercloud vm
+    ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
+        ./test_ovs_rpm.sh $setkmod
+EOI
+fi
+#
+# copy rpms from undercloud back to host
+#
+echo copy rpms from undercloud back to $TMP_RELEASE_DIR in host
+scp ${SSH_OPTIONS[@]} stack@$UNDERCLOUD:rpmbuild/RPMS/x86_64/*.rpm $TMP_RELEASE_DIR
+
+exit 0
diff --git a/build/test_ovs_rpm.sh b/build/test_ovs_rpm.sh
new file mode 100755
index 0000000..fce5ca4
--- /dev/null
+++ b/build/test_ovs_rpm.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc. and others.
+# therbert@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -e
+declare -i CNT
+
+echo "==============================="
+echo executing $0 $@
+echo executing on machine `uname -a`
+
+
+usage() {
+    echo run BuildAndTest -h for help
+}
+
+while getopts "dg:hkp:u:v" opt; do
+    case "$opt" in
+        d)
+            DPDK="yes"
+            ;;
+        g)
+            TAG=${OPTARG}
+            ;;
+        h|\?)
+            usage
+            exit 1
+            ;;
+        k)
+            kmod="yes"
+            ;;
+        u)
+            OVS_REPO_URL=${OPTARG}
+            ;;
+        v)
+            verbose="yes"
+            ;;
+    esac
+done
+
+HOME=`pwd`
+TOPDIR=$HOME
+TMPDIR=$TOPDIR/ovsrpm
+
+if [ -d $TMPDIR ]
+then
+    rm -rf $TMPDIR
+fi
+
+mkdir -p $TMPDIR
+
+cd $TMPDIR
+
+mkdir -p $HOME/rpmbuild/RPMS
+mkdir -p $HOME/rpmbuild/SOURCES
+mkdir -p $HOME/rpmbuild/SPECS
+mkdir -p $HOME/rpmbuild/SRPMS
+
+RPMDIR=$HOME/rpmbuild
+
+echo " Testing installation of kmod RPM"
+if [ ! -z $kmod ]; then
+    echo "Install kernel module"
+    sudo rpm -ivh $RPMDIR/RPMS/x86_64/openvswitch-kmod*.rpm
+    echo " Kernel RPM installed."
+fi
+echo "Testing User Space RPM"
+sudo rpm -ivh $RPMDIR/RPMS/x86_64/openvswitch-2*.rpm
+
+sudo service openvswitch start
+
+sudo ovs-vsctl show
+sudo ovs-vsctl add-br br1
+sudo ovs-ofctl dump-flows br1
+
+exit 0
diff --git a/ci/buildovs.sh b/ci/buildovs.sh
index 77cb96d..db3ac9b 100755
--- a/ci/buildovs.sh
+++ b/ci/buildovs.sh
@@ -1,12 +1,16 @@
 #!/bin/bash
 ##############################################################################
-# Copyright (c) 2015 Red Hat Inc. and others.
+# Copyright (c) 2015,2016 Red Hat Inc. and others.
 # therbert@redhat.com
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+set -e
+
+echo "==================================="
+echo executing $0 $@
 
 # Check to verify that I am being run by Jenkins CI.
 
@@ -16,9 +20,7 @@ if [ -z ${WORKSPACE+1} ]; then
 fi
 
 
-if [ -z ${OVSTAG+1} ]; then
-    export TAG=master
-else
+if [ ${OVSTAG} ]; then
     export TAG=$OVSTAG
 fi
 
@@ -38,7 +40,6 @@ if [ ! -f $BUILD_BASE/config ]; then
 fi
 
 export PATH=$PATH:$WORKSPACE/ci:$BUILD_BASE
-
 source $BUILD_BASE/config
 
 cd $BUILD_BASE
@@ -65,95 +66,26 @@ then
     mkdir -p $TMP_RELEASE_DIR
 fi
 
-# Centos build server should support the following build prerequisites
-
-# yum install gcc make python-devel openssl-devel kernel-devel graphviz \
-# kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \
-# libtool
-
-if [ -d $TMPDIR ]
-then
-    rm -rf $TMPDIR
-fi
-
-mkdir $TMPDIR
-
-cd $TMPDIR
-
-echo "---------------------"
-echo "Clone git repo $TAG"
-echo
-git clone https://github.com/openvswitch/ovs.git
-
-cd ovs
-echo "--------------------"
-echo "Checkout OVS $TAG"
-echo
-if [[ ! "$TAG" =~ "master" ]]; then
-    git checkout $TAG
-fi
-./boot.sh
-./configure
-echo "--------------------"
-echo "Make OVS $TAG"
-echo
-make
 #
-# Get version for master
+# Build ovs rpm without DPDK from ovs master
+#
+echo =======Build ovs rpm and ovs kmod rpm without DPDK Test in VM==========
+    BuildAndTestOVS.sh -p none -t
+#
+# Build ovs rpm with DPDK
+#
+echo =======Build ovs rpm with DPDK Test in VM==========
+BuildAndTestOVS.sh -d -p none -t
+#
+# Build special version of ovs with patches --TODO
 #
-echo "--------------------"
-echo "Get OVS version for $TAG"
-echo
-if [[ "$TAG" =~ "master" ]]; then
-    v=$($TMPDIR/ovs/utilities/ovs-vsctl --version | head -1 | cut -d' ' -f4)
-    export VERSION=$v
-else
-    export VERSION=${TAG:1}
-fi
-
-echo "--------------------"
-echo "OVS version is $VERSION"
-echo
-echo "--------------------"
-echo "Make OVS distribution $TAG"
-echo
-
-make dist
-
-cd $TMPDIR/ovs
-
-cp openvswitch-$VERSION.tar.gz $TOPDIR/rpmbuild/SOURCES
-cp openvswitch-$VERSION.tar.gz $TMPDIR
-
-cd $TMPDIR
-tar -xzf openvswitch-$VERSION.tar.gz
-
-cd $TMPDIR/openvswitch-$VERSION
-
-
-echo "--------------------"
-echo "Build OVS RPM"
-echo
-
-if [ ! -z ${NOCHECK+1} ]; then
-    # Build RPM without checks
-    #
-    rpmbuild -bb --define "_topdir `echo $RPMDIR`" --without check rhel/openvswitch.spec
-else
-    rpmbuild -bb --define "_topdir `echo $RPMDIR`" rhel/openvswitch.spec
-fi
 
 # Once build is done copy product to artifactory.
+# and cleanup
 
-echo "---------------------------------------"
-echo "Copy RPM into $TMP_RELEASE_DIR"
-echo
-cp $RPMDIR/RPMS/x86_64/*.rpm $TMP_RELEASE_DIR
-
-# cleanup
 
 echo "---------------------------------------"
-echo "Cleanup $TMP_RELEASE_DIR"
+echo "Cleanup temporary dirs"
 echo
 cd $BUILDDIR
 
@@ -169,4 +101,17 @@ if [[ "$JOB_NAME" =~ "daily" ]]; then
     upload_artifacts.sh
 fi
 
+if [ -d $TMP_RELEASE_DIR ]; then
+    rm -rf $CACHE_RELEASE_DIR
+fi
+
+if [ -d $RPMDIR ]; then
+    rm -rf $RPMDIR
+fi
+
+# Destroy VM if one has been deployed. Also remove any local installation of
+# DPDK and OVS
+#
+clean.sh
+
 exit 0
diff --git a/ci/clean.sh b/ci/clean.sh
new file mode 100755
index 0000000..b1033b0
--- /dev/null
+++ b/ci/clean.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+##############################################################################
+# Copyright (c) 2016 Red Hat Inc. and others.
+# therbert@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+#Clean script to uninstall provisioning server for Apex
+#author: Dan Radez (dradez@redhat.com)
+#
+vm_index=4
+
+# Clean off instack VM
+virsh destroy instack 2> /dev/null || echo -n ''
+virsh undefine instack --remove-all-storage 2> /dev/null || echo -n ''
+virsh vol-delete instack.qcow2 --pool default 2> /dev/null
+rm -f /var/lib/libvirt/images/instack.qcow2 2> /dev/null
+
+# Clean off baremetal VMs in case they exist
+for i in $(seq 0 $vm_index); do
+  virsh destroy baremetalbrbm_brbm1_$i 2> /dev/null || echo -n ''
+  virsh undefine baremetalbrbm_brbm1_$i --remove-all-storage 2> /dev/null || echo -n ''
+  virsh vol-delete baremetalbrbm_brbm1_${i}.qcow2 --pool default 2> /dev/null
+  rm -f /var/lib/libvirt/images/baremetalbrbm_brbm1_${i}.qcow2 2> /dev/null
+done
+
+# Clean off brbm bridges
+virsh net-destroy brbm 2> /dev/null
+virsh net-undefine brbm 2> /dev/null
+vs-vsctl del-br brbm 2> /dev/null
+
+virsh net-destroy brbm1 2> /dev/null
+virsh net-undefine brbm1 2> /dev/null
+vs-vsctl del-br brbm1 2> /dev/null
+
+# clean pub keys from root's auth keys
+sed -i '/stack@instack.localdomain/d' /root/.ssh/authorized_keys
+sed -i '/virtual-power-key/d' /root/.ssh/authorized_keys
+
+
+echo "Cleanup Completed"