Merge "Allow Asynchronous Apex Builds"

10 files changed, 2179 insertions, 841 deletions

diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh
index b6b2f212a..4e20df96a 100755
--- a/jjb/apex/apex-build.sh
+++ b/jjb/apex/apex-build.sh
@@ -10,7 +10,7 @@ echo
 [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
 # set OPNFV_ARTIFACT_VERSION
 if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then
-  export OPNFV_ARTIFACT_VERSION=dev${BUILD_NUMBER}
+  export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}${GERRIT_PATCHSET_NUMBER}"
   export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY"
 elif echo $BUILD_TAG | grep "csit" 1> /dev/null; then
   export OPNFV_ARTIFACT_VERSION=csit${BUILD_NUMBER}
@@ -23,6 +23,9 @@ else
   export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY --iso"
 fi
 
+# Temporary hack until we fix apex build script
+BUILD_DIRECTORY=${WORKSPACE}/build
+
 # start the build
 cd $WORKSPACE/ci
 ./build.sh $BUILD_ARGS
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh
index 06f7622f5..2e00fae90 100755
--- a/jjb/apex/apex-deploy.sh
+++ b/jjb/apex/apex-deploy.sh
@@ -7,64 +7,10 @@ APEX_PKGS="common undercloud" # removed onos for danube
 IPV6_FLAG=False
 
 # log info to console
-echo "Starting the Apex virtual deployment."
+echo "Starting the Apex deployment."
 echo "--------------------------------------------------------"
 echo
 
-if ! rpm -q wget > /dev/null; then
-  sudo yum -y install wget
-fi
-
-if [[ "$BUILD_DIRECTORY" == *verify* || "$BUILD_DIRECTORY" == *promote* ]]; then
-    # Build is from a verify, use local build artifacts (not RPMs)
-    cd $WORKSPACE/../${BUILD_DIRECTORY}
-    WORKSPACE=$(pwd)
-    echo "WORKSPACE modified to $WORKSPACE"
-    cd $WORKSPACE/ci
-elif [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
-    # if artifact name is passed the pull a
-    # specific artifact from artifacts.opnfv.org
-    # artifact specified should be opnfv-apex-<version>.noarch.rpm
-    RPM_INSTALL_PATH=$GS_URL
-    RPM_LIST=$RPM_INSTALL_PATH/$ARTIFACT_NAME
-else
-    # Use latest RPMS
-    if [[ $BUILD_DIRECTORY == *apex-build* ]]; then
-      # Triggered from a daily so RPMS should be in local directory
-      BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
-      echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
-
-      if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then
-        # if opnfv.properties exists then use the
-        # local build. Source the file so we get local OPNFV vars
-        source ${BUILD_DIRECTORY}/../opnfv.properties
-        RPM_INSTALL_PATH=${BUILD_DIRECTORY}/noarch
-        RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
-      else
-        echo "BUILD_DIRECTORY is from a daily job, so will not use latest from URL"
-        echo "Check that the slave has opnfv.properties in $BUILD_DIRECTORY"
-        exit 1
-      fi
-    else
-      # use the latest from artifacts.opnfv.org
-      # get the latest.properties to get the link to the latest artifact
-      if ! wget -O $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties; then
-        echo "ERROR: Unable to find latest.properties at ${GS_URL}...exiting"
-        exit 1
-      fi
-      # source the file so we get OPNFV vars
-      source opnfv.properties
-      RPM_INSTALL_PATH=$(echo "http://"$OPNFV_RPM_URL | sed 's/\/'"$(basename $OPNFV_RPM_URL)"'//')
-      RPM_LIST=${RPM_INSTALL_PATH}/$(basename $OPNFV_RPM_URL)
-    fi
-fi
-
-# rename odl_l3 to odl only for master
-# this can be removed once all the odl_l3 references
-# are updated to odl after the danube jobs are removed
-if [[ "$BUILD_DIRECTORY" == *master* ]]; then
-    DEPLOY_SCENARIO=${DEPLOY_SCENARIO/odl_l3/odl}
-fi
 if [ -z "$DEPLOY_SCENARIO" ]; then
   echo "Deploy scenario not set!"
   exit 1
@@ -85,75 +31,60 @@ elif [[ "$DEPLOY_SCENARIO" == *gate* ]]; then
   fi
 fi
 
-# use local build for verify and promote
-if [[ "$BUILD_DIRECTORY" == *verify* || "$BUILD_DIRECTORY" == *promote* ]]; then
-    if [ ! -e "${WORKSPACE}/build/lib" ]; then
-      ln -s ${WORKSPACE}/lib ${WORKSPACE}/build/lib
-    fi
-    DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy"
-    NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network"
-    DEPLOY_CMD="$(pwd)/deploy.sh"
-    IMAGES="${WORKSPACE}/.build/"
-    BASE="${WORKSPACE}/build"
-    LIB="${WORKSPACE}/lib"
-    # Make sure python34 deps are installed
-    for dep_pkg in epel-release python34 python34-PyYAML python34-setuptools; do
-      if ! rpm -q ${dep_pkg} > /dev/null; then
-        if ! sudo yum install -y ${dep_pkg}; then
-          echo "Failed to install ${dep_pkg}"
-          exit 1
-        fi
-      fi
-    done
-
-    # Make sure jinja2 is installed
-    for python_pkg in jinja2; do
-      if ! python3.4 -c "import $python_pkg"; then
-        echo "$python_pkg package not found for python3.4, attempting to install..."
-        if ! sudo easy_install-3.4 $python_pkg; then
-          echo -e "Failed to install $python_pkg package for python3.4"
-          exit 1
-        fi
-      fi
-    done
-
-    # Make sure ipxe-roms-qemu package is updated to latest.
-    # This package is needed for multi virtio nic PXE boot in virtual environment.
-    sudo yum update -y ipxe-roms-qemu
-
-    if [ -z ${PYTHONPATH:-} ]; then
-        export PYTHONPATH=${WORKSPACE}/lib/python
-    else
-        export PYTHONPATH=$PYTHONPATH:${WORKSPACE}/lib/python
-    fi
-# use RPMs
+# Dev or RPM/ISO build
+if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
+  # Settings for deploying from git workspace
+  DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy"
+  NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network"
+  DEPLOY_CMD="${WORKSPACE}/ci/deploy.sh"
+  RESOURCES="${WORKSPACE}/.build/"
+  CONFIG="${WORKSPACE}/build"
+  LIB="${WORKSPACE}/lib"
+
+  # Ensure artifacts were downloaded and extracted correctly
+  # TODO(trozet) add verification here
+
 else
-    # find version of RPM
-    VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | grep -Eo '[0-9]+\.[0-9]+-([0-9]{8}|[a-z]+-[0-9]\.[0-9]+)')
-    # build RPM List which already includes base Apex RPM
-    for pkg in ${APEX_PKGS}; do
-        RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}.noarch.rpm"
-    done
-
-    # remove old / install new RPMs
-    if rpm -q opnfv-apex > /dev/null; then
-      INSTALLED_RPMS=$(rpm -qa | grep apex)
-      if [ -n "$INSTALLED_RPMS" ]; then
-        sudo yum remove -y ${INSTALLED_RPMS}
-      fi
-    fi
+  DEPLOY_SETTINGS_DIR="/etc/opnfv-apex/"
+  NETWORK_SETTINGS_DIR="/etc/opnfv-apex/"
+  DEPLOY_CMD="opnfv-deploy"
+  RESOURCES="/var/opt/opnfv/images"
+  CONFIG="/var/opt/opnfv"
+  LIB="/var/opt/opnfv/lib"
+
+fi
 
-    if ! sudo yum install -y $RPM_LIST; then
-      echo "Unable to install new RPMs: $RPM_LIST"
+# Install Dependencies
+# Make sure python34 dependencies are installed
+for dep_pkg in epel-release python34 python34-PyYAML python34-setuptools; do
+  if ! rpm -q ${dep_pkg} > /dev/null; then
+    if ! sudo yum install -y ${dep_pkg}; then
+      echo "Failed to install ${dep_pkg}"
       exit 1
     fi
-
-    DEPLOY_CMD=opnfv-deploy
-    DEPLOY_SETTINGS_DIR="/etc/opnfv-apex/"
-    NETWORK_SETTINGS_DIR="/etc/opnfv-apex/"
-    IMAGES="/var/opt/opnfv/images"
-    BASE="/var/opt/opnfv"
-    LIB="/var/opt/opnfv/lib"
+  fi
+done
+
+# Make sure jinja2 is installed
+for python_pkg in jinja2; do
+  if ! python3.4 -c "import $python_pkg"; then
+    echo "$python_pkg package not found for python3.4, attempting to install..."
+    if ! sudo easy_install-3.4 $python_pkg; then
+      echo -e "Failed to install $python_pkg package for python3.4"
+      exit 1
+    fi
+  fi
+done
+
+if [[ "$JOB_NAME" =~ "virtual" ]]; then
+  # Make sure ipxe-roms-qemu package is updated to latest.
+  # This package is needed for multi virtio nic PXE boot in virtual environment.
+  sudo yum update -y ipxe-roms-qemu
+  if [ -z ${PYTHONPATH:-} ]; then
+    export PYTHONPATH=${WORKSPACE}/lib/python
+  else
+    export PYTHONPATH=$PYTHONPATH:${WORKSPACE}/lib/python
+  fi
 fi
 
 # set env vars to deploy cmd
@@ -165,8 +96,9 @@ if [ "$OPNFV_CLEAN" == 'yes' ]; then
   else
     clean_opts=''
   fi
-  if [[ "$BUILD_DIRECTORY" == *verify* || "$BUILD_DIRECTORY" == *promote* ]]; then
-    sudo BASE=${BASE} LIB=${LIB} ./clean.sh ${clean_opts}
+
+  if [[ "$ARTIFACT_VERSION" =~ "dev" ]]; then
+    sudo CONFIG=${CONFIG} LIB=${LIB} ./clean.sh ${clean_opts}
   else
     sudo BASE=${BASE} LIB=${LIB} opnfv-clean ${clean_opts}
   fi
@@ -185,7 +117,7 @@ if [ ! -e "$DEPLOY_FILE" ]; then
   echo "ERROR: Required settings file missing: Deploy settings file ${DEPLOY_FILE}"
 fi
 
-if [[ "$JOB_NAME" == *virtual* ]]; then
+if [[ "$JOB_NAME" =~ "virtual" ]]; then
   # settings for virtual deployment
   DEPLOY_CMD="${DEPLOY_CMD} -v"
   if [[ "${DEPLOY_SCENARIO}" =~ fdio|ovs ]]; then
@@ -202,17 +134,6 @@ else
   NETWORK_SETTINGS_DIR="/root/network"
   INVENTORY_FILE="/root/inventory/pod_settings.yaml"
 
-# (trozet) According to FDS folks uio_pci_generic works with UCS-B
-# and there appears to be a bug with vfio-pci
-  # if fdio on baremetal, then we are using UCS enic and
-  # need to use vfio-pci instead of uio generic
-#  if [[ "$DEPLOY_SCENARIO" == *fdio* ]]; then
-#    TMP_DEPLOY_FILE="${WORKSPACE}/${DEPLOY_SCENARIO}.yaml"
-#    cp -f ${DEPLOY_FILE} ${TMP_DEPLOY_FILE}
-#    sed -i 's/^\(\s*uio-driver:\).*$/\1 vfio-pci/g' ${TMP_DEPLOY_FILE}
-#    DEPLOY_FILE=${TMP_DEPLOY_FILE}
-#  fi
-
   if ! sudo test -e "$INVENTORY_FILE"; then
     echo "ERROR: Required settings file missing: Inventory settings file ${INVENTORY_FILE}"
     exit 1
diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh
new file mode 100755
index 000000000..a921d61aa
--- /dev/null
+++ b/jjb/apex/apex-download-artifact.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+APEX_PKGS="common undercloud" # removed onos for danube
+
+[[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY
+
+if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
+  # dev build
+  export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}${GERRIT_PATCHSET_NUMBER}"
+  # get build artifact
+  pushd ${BUILD_DIRECTORY} > /dev/null
+  echo "Downloading packaged dev build..."
+  curl --fail -s -o $BUILD_DIRECTORY/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz http://$GS_URL/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz
+  tar -xvf apex-${OPNFV_ARTIFACT_VERSION}.tar.gz
+  popd > /dev/null
+else
+  # Must be RPMs/ISO
+  export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d")
+
+  # get the properties file in order to get info regarding artifacts
+  curl --fail -s -o $BUILD_DIRECTORY/opnfv.properties http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties
+
+  # source the file so we get OPNFV vars
+  source $BUILD_DIRECTORY/opnfv.properties
+
+  RPM_INSTALL_PATH=$(echo "http://"$OPNFV_RPM_URL | sed 's/\/'"$(basename $OPNFV_RPM_URL)"'//')
+  RPM_LIST=${RPM_INSTALL_PATH}/$(basename $OPNFV_RPM_URL)
+
+  # find version of RPM
+  VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | grep -Eo '[0-9]+\.[0-9]+-([0-9]{8}|[a-z]+-[0-9]\.[0-9]+)')
+  # build RPM List which already includes base Apex RPM
+  for pkg in ${APEX_PKGS}; do
+    RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}.noarch.rpm"
+  done
+
+  # remove old / install new RPMs
+  if rpm -q opnfv-apex > /dev/null; then
+    INSTALLED_RPMS=$(rpm -qa | grep apex)
+    if [ -n "$INSTALLED_RPMS" ]; then
+      sudo yum remove -y ${INSTALLED_RPMS}
+    fi
+  fi
+  if ! sudo yum install -y $RPM_LIST; then
+    echo "Unable to install new RPMs: $RPM_LIST"
+    exit 1
+  fi
+
+  # log info to console
+  echo "Downloading the ISO artifact using URL http://$OPNFV_ARTIFACT_URL"
+  echo "--------------------------------------------------------"
+  echo
+
+  # Download ISO
+  curl --fail -s -o $BUILD_DIRECTORY/apex.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
+
+fi
+
+# TODO: Uncomment these lines to verify SHA512SUMs once the sums are
+# fixed.
+# echo "$OPNFV_ARTIFACT_SHA512SUM $BUILD_DIRECTORY/apex.iso" | sha512sum -c
+# echo "$OPNFV_RPM_SHA512SUM $BUILD_DIRECTORY/$(basename $OPNFV_RPM_URL)" | sha512sum -c
+
+# list the files
+ls -al $BUILD_DIRECTORY
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/apex/apex-jjb-renderer.py b/jjb/apex/apex-jjb-renderer.py
new file mode 100644
index 000000000..da62bc7ad
--- /dev/null
+++ b/jjb/apex/apex-jjb-renderer.py
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pprint
+import yaml
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+
+gspathname = dict()
+branch = dict()
+env = Environment(loader=FileSystemLoader('./'), autoescape=True)
+
+with open('scenarios.yaml.hidden') as _:
+    scenarios = yaml.safe_load(_)
+
+template = env.get_template('apex.yml.j2')
+
+print("Scenarios are: ")
+pprint.pprint(scenarios)
+
+for stream in scenarios:
+    if stream == 'master':
+        gspathname['master'] = ''
+        branch[stream] = stream
+    else:
+        gspathname[stream] = '/' + stream
+        branch[stream] = 'stable/' + stream
+
+output = template.render(scenarios=scenarios, gspathname=gspathname,
+                         branch=branch)
+
+with open('./apex.yml', 'w') as fh:
+    fh.write(output)
diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh
index d046c119d..96c22bfac 100755
--- a/jjb/apex/apex-upload-artifact.sh
+++ b/jjb/apex/apex-upload-artifact.sh
@@ -18,35 +18,37 @@ source $WORKSPACE/opnfv.properties
 
 BUILD_DIRECTORY=${WORKSPACE}/.build
 
-# clone releng repository
-echo "Cloning releng repository..."
-[ -d releng ] && rm -rf releng
-git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null
-#this is where we import the siging key
-if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then
-  source $WORKSPACE/releng/utils/gpg_import_key.sh
-fi
+importkey () {
+  # clone releng repository
+  echo "Cloning releng repository..."
+  [ -d releng ] && rm -rf releng
+  git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null
+  #this is where we import the siging key
+  if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then
+    source $WORKSPACE/releng/utils/gpg_import_key.sh
+  fi
+}
 
 signrpm () {
-for artifact in $RPM_LIST $SRPM_LIST; do
-  echo "Signing artifact: ${artifact}"
-  gpg2 -vvv --batch --yes --no-tty \
-    --default-key opnfv-helpdesk@rt.linuxfoundation.org \
-    --passphrase besteffort \
-    --detach-sig $artifact
-    gsutil cp "$artifact".sig gs://$GS_URL/$(basename "$artifact".sig)
-    echo "Upload complete for ${artifact} signature"
-done
+  for artifact in $RPM_LIST $SRPM_LIST; do
+    echo "Signing artifact: ${artifact}"
+    gpg2 -vvv --batch --yes --no-tty \
+      --default-key opnfv-helpdesk@rt.linuxfoundation.org \
+      --passphrase besteffort \
+      --detach-sig $artifact
+      gsutil cp "$artifact".sig gs://$GS_URL/$(basename "$artifact".sig)
+      echo "Upload complete for ${artifact} signature"
+  done
 }
 
 signiso () {
-time gpg2 -vvv --batch --yes --no-tty \
-  --default-key opnfv-helpdesk@rt.linuxfoundation.org  \
-  --passphrase besteffort \
-  --detach-sig $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso
+  gpg2 -vvv --batch --yes --no-tty \
+    --default-key opnfv-helpdesk@rt.linuxfoundation.org  \
+    --passphrase besteffort \
+    --detach-sig $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso
 
-gsutil cp $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso.sig 
-echo "ISO signature Upload Complete!"
+  gsutil cp $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso.sig
+  echo "ISO signature Upload Complete!"
 }
 
 uploadiso () {
@@ -62,6 +64,13 @@ uploadrpm () {
   done
   gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log
   gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log
+
+  # Make the property files viewable on the artifact site
+  gsutil -m setmeta \
+    -h "Content-Type:text/html" \
+    -h "Cache-Control:private, max-age=0, no-transform" \
+    gs://$GS_URL/latest.properties \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1
 }
 
 uploadsnap () {
@@ -75,6 +84,20 @@ uploadsnap () {
   echo "Upload complete for Snapshot"
 }
 
+uploadimages () {
+  # Uploads dev tarball
+  export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}${GERRIT_PATCHSET_NUMBER}"
+  echo "Uploading development build tarball"
+  pushd $BUILD_DIRECTORY > /dev/null
+  tar czf apex-${OPNFV_ARTIFACT_VERSION}.tar.gz *.qcow2
+  gsutil cp apex-${OPNFV_ARTIFACT_VERSION}.tar.gz gs://$GS_URL/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz > gsutil.latest.log
+  popd > /dev/null
+}
+
+# Always import the signing key, if it's available the artifacts will be
+# signed before being uploaded
+importkey
+
 if gpg2 --list-keys | grep "opnfv-helpdesk@rt.linuxfoundation.org"; then
   echo "Signing Key avaliable"
   SIGN_ARTIFACT="true"
@@ -83,28 +106,38 @@ fi
 if [ "$ARTIFACT_TYPE" == 'snapshot' ]; then
   uploadsnap
 elif [ "$ARTIFACT_TYPE" == 'iso' ]; then
+  if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
+    echo "Skipping artifact upload for ${ARTIFACT_TYPE} due to dev build"
+    exit 0
+  fi
   if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then
     signiso
   fi
   uploadiso
 elif [ "$ARTIFACT_TYPE" == 'rpm' ]; then
-  RPM_INSTALL_PATH=$BUILD_DIRECTORY/noarch
-  RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
-  VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//')
-  for pkg in common undercloud; do # removed onos for danube
-    RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
-  done
-  SRPM_INSTALL_PATH=$BUILD_DIRECTORY
-  SRPM_LIST=$SRPM_INSTALL_PATH/$(basename $OPNFV_SRPM_URL)
-  VERSION_EXTENSION=$(echo $(basename $OPNFV_SRPM_URL) | sed 's/opnfv-apex-//')
-  for pkg in common undercloud; do # removed onos for danube
-    SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
-  done
+  if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
+    echo "dev build detected, will upload image tarball"
+    ARTIFACT_TYPE=tarball
+    uploadimages
+  else
+    RPM_INSTALL_PATH=$BUILD_DIRECTORY/noarch
+    RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
+    VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//')
+    for pkg in common undercloud; do # removed onos for danube
+      RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
+    done
+    SRPM_INSTALL_PATH=$BUILD_DIRECTORY
+    SRPM_LIST=$SRPM_INSTALL_PATH/$(basename $OPNFV_SRPM_URL)
+    VERSION_EXTENSION=$(echo $(basename $OPNFV_SRPM_URL) | sed 's/opnfv-apex-//')
+    for pkg in common undercloud; do # removed onos for danube
+      SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
+    done
 
-  if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then
-    signrpm
+    if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then
+      signrpm
+    fi
+    uploadrpm
   fi
-  uploadrpm
 else
   echo "ERROR: Unknown artifact type ${ARTIFACT_TYPE} to upload...exiting"
   exit 1
@@ -115,3 +148,4 @@ echo "--------------------------------------------------------"
 echo "Done!"
 if [ "$ARTIFACT_TYPE" == 'iso' ]; then echo "ISO Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"; fi
 if [ "$ARTIFACT_TYPE" == 'rpm' ]; then echo "RPM Artifact is available as http://$GS_URL/$(basename $OPNFV_RPM_URL)"; fi
+if [ "$ARTIFACT_TYPE" == 'tarball' ]; then echo "Dev tarball Artifact is available as http://$GS_URL/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz)"; fi
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml
index 7ca2e6edd..699bdb3b8 100644
--- a/jjb/apex/apex.yml
+++ b/jjb/apex/apex.yml
@@ -1,65 +1,41 @@
 - project:
-    name: apex
+    name: 'apex'
+    project: 'apex'
+    build-slave: 'apex-build'
     jobs:
         - 'apex-verify-{stream}'
         - 'apex-verify-gate-{stream}'
         - 'apex-verify-unit-tests-{stream}'
-        - 'apex-runner-{platform}-{scenario}-{stream}'
         - 'apex-runner-cperf-{stream}'
         - 'apex-build-{stream}'
-        - 'apex-deploy-virtual-{scenario}-{stream}'
-        - 'apex-deploy-baremetal-{scenario}-{stream}'
-        - 'apex-daily-{stream}'
+        - 'apex-deploy-{platform}-{stream}'
+        - 'apex-daily-master'
+        - 'apex-daily-danube'
         - 'apex-csit-promote-daily-{stream}'
         - 'apex-fdio-promote-daily-{stream}'
         - 'apex-verify-iso-{stream}'
-
+        - 'apex-deploy-test-baremetal-{stream}'
+        - 'apex-upload-snapshot'
+        - 'apex-create-snapshot'
     # stream:    branch with - in place of / (eg. stable-arno)
     # branch:    branch (eg. stable/arno)
     stream:
         - master:
             branch: 'master'
             gs-pathname: ''
-            slave: 'lf-pod1'
-            verify-slave: 'apex-verify-master'
-            daily-slave: 'apex-daily-master'
+            build-slave: 'apex-build-master'
+            virtual-slave: 'apex-virtual-master'
+            baremetal-slave: 'apex-baremetal-master'
+            verify-scenario: 'os-odl-nofeature-ha'
+
         - danube:
             branch: 'stable/danube'
             gs-pathname: '/danube'
-            slave: 'lf-pod1'
-            verify-slave: 'apex-verify-danube'
-            daily-slave: 'apex-daily-danube'
-
-    project: 'apex'
-
-    scenario:
-         - 'os-nosdn-nofeature-noha'
-         - 'os-nosdn-nofeature-ha'
-         - 'os-nosdn-nofeature-ha-ipv6'
-         - 'os-nosdn-ovs-noha'
-         - 'os-nosdn-ovs-ha'
-         - 'os-nosdn-fdio-noha'
-         - 'os-nosdn-fdio-ha'
-         - 'os-nosdn-kvm-ha'
-         - 'os-nosdn-kvm-noha'
-         - 'os-odl_l2-fdio-noha'
-         - 'os-odl_l2-fdio-ha'
-         - 'os-odl_l2-netvirt_gbp_fdio-noha'
-         - 'os-odl_l2-sfc-noha'
-         - 'os-odl_l3-nofeature-noha'
-         - 'os-odl_l3-nofeature-ha'
-         - 'os-odl_l3-ovs-noha'
-         - 'os-odl_l3-ovs-ha'
-         - 'os-odl-bgpvpn-ha'
-         - 'os-odl-gluon-noha'
-         - 'os-odl_l3-fdio-noha'
-         - 'os-odl_l3-fdio-ha'
-         - 'os-odl_l3-fdio_dvr-noha'
-         - 'os-odl_l3-fdio_dvr-ha'
-         - 'os-odl_l3-csit-noha'
-         - 'os-onos-nofeature-ha'
-         - 'os-ovn-nofeature-noha'
-         - 'gate'
+            build-slave: 'apex-build-danube'
+            virtual-slave: 'apex-virtual-danube'
+            baremetal-slave: 'apex-baremetal-danube'
+            verify-scenario: 'os-odl_l3-nofeature-ha'
+            disabled: false
 
     platform:
          - 'baremetal'
@@ -69,7 +45,7 @@
 - job-template:
     name: 'apex-verify-unit-tests-{stream}'
 
-    node: '{verify-slave}'
+    node: '{build-slave}'
 
     concurrent: true
 
@@ -79,10 +55,6 @@
         - project-parameter:
             project: '{project}'
             branch: '{branch}'
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: "Used for overriding the GIT URL coming from parameters macro."
 
     scm:
         - git-scm-gerrit
@@ -117,27 +89,29 @@
             option: 'project'
 
     builders:
-        - 'apex-unit-test'
+        - shell:
+            !include-raw-escape: ./apex-unit-test.sh
 
 # Verify
 - job-template:
     name: 'apex-verify-{stream}'
 
-    node: '{verify-slave}'
+    node: '{virtual-slave}'
 
     concurrent: true
 
+    project-type: 'multijob'
+
     parameters:
         - apex-parameter:
-            gs-pathname: '{gs-pathname}'
+            gs-pathname: '{gs-pathname}/dev'
         - project-parameter:
             project: '{project}'
             branch: '{branch}'
         - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: "Used for overriding the GIT URL coming from parameters macro."
-
+            name: ARTIFACT_VERSION
+            default: dev
+            description: "Used for overriding the ARTIFACT_VERSION"
     scm:
         - git-scm-gerrit
 
@@ -178,7 +152,6 @@
             blocking-jobs:
                 - 'apex-daily.*'
                 - 'apex-deploy.*'
-                - 'apex-build.*'
                 - 'apex-runner.*'
                 - 'apex-verify.*'
         - throttle:
@@ -187,36 +160,86 @@
             option: 'project'
 
     builders:
-        - 'apex-unit-test'
-        - 'apex-build'
-        - trigger-builds:
-          - project: 'apex-deploy-virtual-os-odl_l3-nofeature-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-verify-{stream}
-              OPNFV_CLEAN=yes
-            git-revision: false
-            block: true
-            same-node: true
-        - trigger-builds:
-          - project: 'functest-apex-{verify-slave}-suite-{stream}'
-            predefined-parameters: |
-              DEPLOY_SCENARIO=os-odl_l3-nofeature-ha
-              FUNCTEST_SUITE_NAME=healthcheck
-            block: true
-            same-node: true
-        - 'apex-workspace-cleanup'
+        - description-setter:
+            description: "Built on $NODE_NAME"
+        - multijob:
+            name: basic
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-verify-unit-tests-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO={verify-scenario}
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO={verify-scenario}
+                  FUNCTEST_SUITE_NAME=healthcheck
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: false
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
 
 # Verify Scenario Gate
 - job-template:
     name: 'apex-verify-gate-{stream}'
 
-    node: '{verify-slave}'
+    node: '{virtual-slave}'
 
     concurrent: true
 
+    project-type: 'multijob'
+
     parameters:
         - apex-parameter:
-            gs-pathname: '{gs-pathname}'
+            gs-pathname: '{gs-pathname}/dev'
         - project-parameter:
             project: '{project}'
             branch: '{branch}'
@@ -267,88 +290,46 @@
             option: 'project'
 
     builders:
-        - 'apex-build'
-        - trigger-builds:
-          - project: 'apex-deploy-virtual-gate-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-verify-gate-{stream}
-              OPNFV_CLEAN=yes
-            current-parameters: true
-            git-revision: false
-            block: true
-            same-node: true
-        - trigger-builds:
-          - project: 'functest-apex-{verify-slave}-suite-{stream}'
-            predefined-parameters: |
-              DEPLOY_SCENARIO=os-nosdn-nofeature-ha
-              FUNCTEST_SUITE_NAME=healthcheck
-            block: true
-            same-node: true
-        - 'apex-workspace-cleanup'
-
-- job-template:
-    name: 'apex-runner-{platform}-{scenario}-{stream}'
-
-    # runner jobs for deploying manually
-
-    node: '{slave}'
-
-    disabled: false
-
-    parameters:
-        - apex-parameter:
-            gs-pathname: '{gs-pathname}'
-        - project-parameter:
-            project: '{project}'
-            branch: '{branch}'
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: "Used for overriding the GIT URL coming from parameters macro."
-
-    scm:
-        - git-scm
-
-    properties:
-        - logrotate-default
-        - build-blocker:
-            use-build-blocker: true
-            blocking-jobs:
-                - 'apex-daily.*'
-                - 'apex-verify.*'
-                - 'apex-.*-promote.*'
-
-    builders:
-        - trigger-builds:
-          - project: 'apex-deploy-{platform}-{scenario}-{stream}'
-            predefined-parameters:
-              OPNFV_CLEAN=yes
-            git-revision: false
-            block: true
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO={scenario}
-            block: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'functest-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO={scenario}
-            block: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=gate
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO={verify-scenario}
+                  FUNCTEST_SUITE_NAME=healthcheck
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: false
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
 
 - job-template:
     name: 'apex-runner-cperf-{stream}'
 
     # runner cperf job
-
+    project-type: 'multijob'
     node: 'intel-pod2'
 
     disabled: false
@@ -380,20 +361,35 @@
             option: 'project'
 
     builders:
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl_l3-nofeature-noha-{stream}'
-            predefined-parameters:
-              OPNFV_CLEAN=yes
-            git-revision: false
-            block: true
-            same-node: true
-        - trigger-builds:
-          - project: 'cperf-apex-intel-pod2-daily-master'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-nofeature-noha
-            block: true
-            same-node: true
-
+        - description-setter:
+            description: "Deployed on $NODE_NAME"
+        - multijob:
+            name: 'Baremetal Deploy'
+            condition: ALWAYS
+            projects:
+                - name: 'apex-deploy-baremetal-{stream}'
+                  node-parameters: false
+                  current-parameters: true
+                  predefined-parameters: |
+                    OPNFV_CLEAN=yes
+                    DEPLOY_SCENARIO={verify-scenario}
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: Functest
+            condition: ALWAYS
+            projects:
+                - name: 'functest-apex-baremetal-daily-{stream}'
+                  node-parameters: true
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO={verify-scenario}
+                  kill-phase-on: NEVER
+                  abort-all-job: false
+                  git-revision: false
+
+# Build phase
 - job-template:
     name: 'apex-build-{stream}'
 
@@ -402,33 +398,25 @@
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{daily-slave}'
+    node: '{build-slave}'
 
     disabled: false
 
     concurrent: true
 
     parameters:
+        - '{project}-defaults'
         - project-parameter:
             project: '{project}'
             branch: '{branch}'
         - apex-parameter:
             gs-pathname: '{gs-pathname}'
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: "Used for overriding the GIT URL coming from parameters macro."
 
     scm:
         - git-scm
 
     properties:
         - logrotate-default
-        - build-blocker:
-            use-build-blocker: true
-            block-level: 'NODE'
-            blocking-jobs:
-                - 'apex-deploy.*'
         - throttle:
             max-per-node: 1
             max-total: 10
@@ -436,27 +424,9 @@
 
     builders:
         - 'apex-build'
-        - trigger-builds:
-          - project: 'apex-deploy-virtual-os-nosdn-nofeature-noha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: false
-            same-node: true
-            block: true
         - inject:
            properties-content: ARTIFACT_TYPE=rpm
         - 'apex-upload-artifact'
-        - trigger-builds:
-          - project: 'apex-verify-iso-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-            git-revision: false
-            block: true
-            same-node: true
-        - inject:
-           properties-content: ARTIFACT_TYPE=iso
-        - 'apex-upload-artifact'
 
 # ISO verify job
 - job-template:
@@ -467,7 +437,7 @@
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{daily-slave}'
+    node: '{virtual-slave}'
 
     disabled: false
 
@@ -501,16 +471,20 @@
 
     builders:
         - 'apex-iso-verify'
+        - inject:
+           properties-content: ARTIFACT_TYPE=iso
+        - 'apex-upload-artifact'
 
+# Deploy job
 - job-template:
-    name: 'apex-deploy-virtual-{scenario}-{stream}'
+    name: 'apex-deploy-{platform}-{stream}'
 
     # Job template for virtual deployment
     #
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{slave}'
+    node: 'apex-{platform}-{stream}'
 
     concurrent: true
 
@@ -527,7 +501,7 @@
             gs-pathname: '{gs-pathname}'
         - string:
             name: DEPLOY_SCENARIO
-            default: '{scenario}'
+            default: '{verify-scenario}'
             description: "Scenario to deploy with."
         - string:
             name: OPNFV_CLEAN
@@ -547,18 +521,23 @@
             option: 'project'
 
     builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
+        - 'apex-download-artifact'
         - 'apex-deploy'
         - 'apex-workspace-cleanup'
 
+
+# Baremetal Deploy and Test
 - job-template:
-    name: 'apex-deploy-baremetal-{scenario}-{stream}'
+    name: 'apex-deploy-test-baremetal-{stream}'
 
-    # Job template for baremetal deployment
+    # Job template for daily build
     #
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{slave}'
+    project-type: 'multijob'
 
     disabled: false
 
@@ -566,6 +545,8 @@
         - git-scm
 
     parameters:
+        - '{project}-defaults'
+        - '{project}-baremetal-{stream}-defaults'
         - project-parameter:
             project: '{project}'
             branch: '{branch}'
@@ -573,8 +554,85 @@
             gs-pathname: '{gs-pathname}'
         - string:
             name: DEPLOY_SCENARIO
-            default: '{scenario}'
+            default: '{verify-scenario}'
             description: "Scenario to deploy with."
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-verify.*'
+                - 'apex-deploy.*'
+                - 'apex-runner.*'
+                - 'apex-.*-promote.*'
+    builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
+        - multijob:
+            name: 'Baremetal Deploy'
+            condition: ALWAYS
+            projects:
+                - name: 'apex-deploy-baremetal-{stream}'
+                  node-parameters: true
+                  current-parameters: true
+                  predefined-parameters: |
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: Functest
+            condition: ALWAYS
+            projects:
+                - name: 'functest-apex-baremetal-daily-{stream}'
+                  node-parameters: true
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                  kill-phase-on: NEVER
+                  abort-all-job: false
+                  git-revision: false
+        - multijob:
+            name: Yardstick
+            condition: ALWAYS
+            projects:
+                - name: 'yardstick-apex-baremetal-daily-{stream}'
+                  node-parameters: true
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                  kill-phase-on: NEVER
+                  abort-all-job: false
+                  git-revision: false
+
+
+# danube Daily
+- job-template:
+    name: 'apex-daily-danube'
+
+    # Job template for daily build
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    project-type: 'multijob'
+
+    node: '{build-slave}'
+
+    disabled: false
+
+    scm:
+        - git-scm
+
+    parameters:
+        - '{project}-defaults'
+        - '{project}-baremetal-danube-defaults'
+        - project-parameter:
+            project: '{project}'
+            branch: 'stable/danube'
+        - apex-parameter:
+            gs-pathname: '/danube'
 
     properties:
         - logrotate-default
@@ -584,23 +642,322 @@
             blocking-jobs:
                 - 'apex-verify.*'
                 - 'apex-deploy.*'
-                - 'apex-build.*'
+                - 'apex-runner.*'
+                - 'apex-.*-promote.*'
 
+    triggers:
+        - 'apex-danube'
 
     builders:
-        - 'apex-deploy'
-        - 'apex-workspace-cleanup'
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-danube'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: 'Verify and upload ISO'
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-verify-iso-danube'
+                  current-parameters: false
+                  predefined-parameters: |
+                    BUILD_DIRECTORY=apex-build-danube/.build
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: Baremetal Deploy and Test Phase
+            condition: SUCCESSFUL
+            projects:
 
-# Daily
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-nofeature-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-nofeature-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-nofeature-ha-ipv6
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-ovs-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-ovs-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-fdio-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-fdio-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-kvm-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-kvm-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l2-fdio-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l2-fdio-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l2-netvirt_gbp_fdio-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l2-sfc-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-nofeature-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-nofeature-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-ovs-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-ovs-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl-bgpvpn-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl-gluon-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-fdio-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-fdio-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-fdio_dvr-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-fdio_dvr-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-csit-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-onos-nofeature-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-danube'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-ovn-nofeature-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+
+# master Daily
 - job-template:
-    name: 'apex-daily-{stream}'
+    name: 'apex-daily-master'
 
     # Job template for daily build
     #
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{daily-slave}'
+    project-type: 'multijob'
+
+    node: '{build-slave}'
 
     disabled: false
 
@@ -608,11 +965,13 @@
         - git-scm
 
     parameters:
+        - '{project}-defaults'
+        - '{project}-baremetal-master-defaults'
         - project-parameter:
             project: '{project}'
-            branch: '{branch}'
+            branch: 'master'
         - apex-parameter:
-            gs-pathname: '{gs-pathname}'
+            gs-pathname: ''
 
     properties:
         - logrotate-default
@@ -622,397 +981,121 @@
             blocking-jobs:
                 - 'apex-verify.*'
                 - 'apex-deploy.*'
-                - 'apex-build.*'
                 - 'apex-runner.*'
                 - 'apex-.*-promote.*'
 
     triggers:
-        - 'apex-{stream}'
+        - 'apex-master'
+
+    builders:
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-master'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: 'Verify and upload ISO'
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-verify-iso-master'
+                  current-parameters: false
+                  predefined-parameters: |
+                    BUILD_DIRECTORY=apex-build-master/.build
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: Baremetal Deploy and Test Phase
+            condition: SUCCESSFUL
+            projects:
+
+                - name: 'apex-deploy-test-baremetal-master'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-nofeature-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-master'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-nosdn-nofeature-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-master'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl-nofeature-ha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+                - name: 'apex-deploy-test-baremetal-master'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl-nofeature-noha
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+
+
+
+# snapshot create
+- job-template:
+    name: 'apex-create-snapshot'
+
+    # Job template for clean
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+
+    disabled: false
 
     builders:
-        - trigger-builds:
-          - project: 'apex-build-{stream}'
-            git-revision: true
-            current-parameters: true
-            same-node: true
-            block: true
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-nosdn-nofeature-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-nofeature-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-nofeature-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        # 1.dovetail only master by now, not sync with A/B/C branches
-        # 2.here the stream means the SUT stream, dovetail stream is defined in its own job
-        # 3.only debug testsuite here(includes basic testcase,
-        #   i.e. one tempest smoke ipv6, two vping from functest)
-        # 4.not used for release criteria or compliance,
-        #   only to debug the dovetail tool bugs with apex
-        #- trigger-builds:
-        #    - project: 'dovetail-apex-{slave}-proposed_tests-{stream}'
-        #      current-parameters: false
-        #      predefined-parameters:
-        #        DEPLOY_SCENARIO=os-nosdn-nofeature-ha
-        #      block: true
-        #      same-node: true
-        #      block-thresholds:
-        #        build-step-failure-threshold: 'never'
-        #        failure-threshold: 'never'
-        #        unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl_l3-nofeature-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-nofeature-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-nofeature-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl-bgpvpn-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl-bgpvpn-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl-bgpvpn-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl-gluon-noha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl-gluon-noha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl-gluon-noha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl_l2-fdio-noha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l2-fdio-noha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l2-fdio-noha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl_l2-fdio-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l2-fdio-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l2-fdio-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-nosdn-kvm-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-kvm-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-kvm-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl_l3-fdio-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-fdio-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-fdio-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-nosdn-fdio-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-fdio-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-fdio-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-nosdn-ovs-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-ovs-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-nosdn-ovs-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-odl_l3-ovs-ha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-ovs-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-odl_l3-ovs-ha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'apex-deploy-baremetal-os-ovn-nofeature-noha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-build-{stream}/.build
-              OPNFV_CLEAN=yes
-            git-revision: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-            block: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-ovn-nofeature-noha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-        - trigger-builds:
-          - project: 'yardstick-apex-{slave}-daily-{stream}'
-            predefined-parameters:
-              DEPLOY_SCENARIO=os-ovn-nofeature-noha
-            block: true
-            same-node: true
-            block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
+        - shell:
+            !include-raw-escape: ./apex-snapshot-create.sh
+
+# snapshot upload
+- job-template:
+    name: 'apex-upload-snapshot'
+
+    # Job template for clean
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+
+    disabled: false
+
+    builders:
+        - inject:
+           properties-content: ARTIFACT_TYPE=snapshot
+        - 'apex-upload-artifact'
 
 # CSIT promote
 - job-template:
@@ -1023,8 +1106,8 @@
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{daily-slave}'
-
+    node: '{virtual-slave}'
+    project-type: 'multijob'
     disabled: false
 
     scm:
@@ -1052,27 +1135,75 @@
         - timed: '0 12 * * 0'
 
     builders:
-        - 'apex-build'
-        - trigger-builds:
-          - project: 'apex-deploy-virtual-os-odl_l3-csit-noha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-csit-promote-daily-{stream}
-              OPNFV_CLEAN=yes
-            git-revision: false
-            block: true
-            same-node: true
-        - trigger-builds:
-          - project: 'functest-apex-{daily-slave}-suite-{stream}'
-            predefined-parameters: |
-              DEPLOY_SCENARIO=os-odl_l3-nofeature-noha
-              FUNCTEST_SUITE_NAME=tempest_smoke_serial
-            block: true
-            same-node: true
-        - shell:
-            !include-raw-escape: ./apex-snapshot-create.sh
-        - inject:
-           properties-content: ARTIFACT_TYPE=snapshot
-        - 'apex-upload-artifact'
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-csit-noha
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO=os-odl_l3-nofeature-noha
+                  FUNCTEST_SUITE_NAME=tempest_smoke_serial
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+        - multijob:
+            name: create snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+        - multijob:
+            name: upload snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
 
 # FDIO promote
 - job-template:
@@ -1083,8 +1214,8 @@
     # Required Variables:
     #     stream:    branch with - in place of / (eg. stable)
     #     branch:    branch (eg. stable)
-    node: '{daily-slave}'
-
+    node: '{virtual-slave}'
+    project-type: 'multijob'
     disabled: false
 
     scm:
@@ -1109,20 +1240,58 @@
                 - 'apex-daily.*'
 
     builders:
-        - 'apex-build'
-        - trigger-builds:
-          - project: 'apex-deploy-virtual-os-odl_l2-fdio-noha-{stream}'
-            predefined-parameters: |
-              BUILD_DIRECTORY=apex-fdio-promote-daily-{stream}
-              OPNFV_CLEAN=yes
-            git-revision: false
-            block: true
-            same-node: true
-        - shell:
-            !include-raw-escape: ./apex-snapshot-create.sh
-        - inject:
-           properties-content: ARTIFACT_TYPE=snapshot
-        - 'apex-upload-artifact'
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l2-netvirt_gbp_fdio-noha
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: create snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+        - multijob:
+            name: upload snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
 
 - job-template:
     name: 'apex-gs-clean-{stream}'
@@ -1148,7 +1317,6 @@
     triggers:
         - 'apex-gs-clean-{stream}'
 
-
 ########################
 # parameter macros
 ########################
@@ -1165,7 +1333,7 @@
             description: "Artifact version type"
         - string:
             name: BUILD_DIRECTORY
-            default: $WORKSPACE/build
+            default: $WORKSPACE/.build
             description: "Directory where the build artifact will be located upon the completion of the build."
         - string:
             name: CACHE_DIRECTORY
@@ -1181,7 +1349,7 @@
             description: "Version directory where opnfv artifacts are stored in gs repository"
         - string:
             name: GS_URL
-            default: artifacts.opnfv.org/$PROJECT{gs-pathname}
+            default: $GS_BASE{gs-pathname}
             description: "URL to Google Storage."
 
 ########################
@@ -1219,6 +1387,12 @@
             !include-raw: ./apex-upload-artifact.sh
 
 - builder:
+    name: 'apex-download-artifact'
+    builders:
+        - shell:
+            !include-raw: ./apex-download-artifact.sh
+
+- builder:
     name: 'apex-gs-cleanup'
     builders:
         - shell:
@@ -1230,7 +1404,6 @@
         - shell:
             !include-raw: ./apex-deploy.sh
 
-
 #######################
 # trigger macros
 ########################
@@ -1245,4 +1418,4 @@
 - trigger:
     name: 'apex-gs-clean-{stream}'
     triggers:
-        - timed: '0 2 * * *'
+        - timed: '0 2 * * *'
\ No newline at end of file
diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2
new file mode 100644
index 000000000..ee69f33e1
--- /dev/null
+++ b/jjb/apex/apex.yml.j2
@@ -0,0 +1,1052 @@
+- project:
+    name: 'apex'
+    project: 'apex'
+    build-slave: 'apex-build'
+    jobs:
+        - 'apex-verify-{stream}'
+        - 'apex-verify-gate-{stream}'
+        - 'apex-verify-unit-tests-{stream}'
+        - 'apex-runner-cperf-{stream}'
+        - 'apex-build-{stream}'
+        - 'apex-deploy-{platform}-{stream}'
+        - 'apex-daily-master'
+        - 'apex-daily-danube'
+        - 'apex-csit-promote-daily-{stream}'
+        - 'apex-fdio-promote-daily-{stream}'
+        - 'apex-verify-iso-{stream}'
+        - 'apex-deploy-test-baremetal-{stream}'
+        - 'apex-upload-snapshot'
+        - 'apex-create-snapshot'
+    # stream:    branch with - in place of / (eg. stable-arno)
+    # branch:    branch (eg. stable/arno)
+    stream:
+        - master:
+            branch: 'master'
+            gs-pathname: ''
+            build-slave: 'apex-build-master'
+            virtual-slave: 'apex-virtual-master'
+            baremetal-slave: 'apex-baremetal-master'
+            verify-scenario: 'os-odl-nofeature-ha'
+
+        - danube:
+            branch: 'stable/danube'
+            gs-pathname: '/danube'
+            build-slave: 'apex-build-danube'
+            virtual-slave: 'apex-virtual-danube'
+            baremetal-slave: 'apex-baremetal-danube'
+            verify-scenario: 'os-odl_l3-nofeature-ha'
+            disabled: false
+
+    platform:
+         - 'baremetal'
+         - 'virtual'
+
+# Unit Test
+- job-template:
+    name: 'apex-verify-unit-tests-{stream}'
+
+    node: '{build-slave}'
+
+    concurrent: true
+
+    parameters:
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+
+    scm:
+        - git-scm-gerrit
+
+    triggers:
+        - gerrit:
+            server-name: 'gerrit.opnfv.org'
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'true'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: 'apex'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: 'tests/**'
+    properties:
+        - logrotate-default
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - shell:
+            !include-raw-escape: ./apex-unit-test.sh
+
+# Verify
+- job-template:
+    name: 'apex-verify-{stream}'
+
+    node: '{virtual-slave}'
+
+    concurrent: true
+
+    project-type: 'multijob'
+
+    parameters:
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}/dev'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - string:
+            name: ARTIFACT_VERSION
+            default: dev
+            description: "Used for overriding the ARTIFACT_VERSION"
+    scm:
+        - git-scm-gerrit
+
+    triggers:
+        - gerrit:
+            server-name: 'gerrit.opnfv.org'
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'true'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: 'apex'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: 'ci/**'
+                  - compare-type: ANT
+                    pattern: 'build/**'
+                  - compare-type: ANT
+                    pattern: 'lib/**'
+                  - compare-type: ANT
+                    pattern: 'config/**'
+
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-daily.*'
+                - 'apex-deploy.*'
+                - 'apex-runner.*'
+                - 'apex-verify.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - description-setter:
+            description: "Built on $NODE_NAME"
+        - multijob:
+            name: basic
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-verify-unit-tests-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO={verify-scenario}
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO={verify-scenario}
+                  FUNCTEST_SUITE_NAME=healthcheck
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: false
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+
+# Verify Scenario Gate
+- job-template:
+    name: 'apex-verify-gate-{stream}'
+
+    node: '{virtual-slave}'
+
+    concurrent: true
+
+    project-type: 'multijob'
+
+    parameters:
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}/dev'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: "Used for overriding the GIT URL coming from parameters macro."
+
+    scm:
+        - git-scm-gerrit
+
+    triggers:
+        - gerrit:
+            server-name: 'gerrit.opnfv.org'
+            trigger-on:
+                - comment-added-contains-event:
+                    comment-contains-value: '^Patch Set [0-9]+: Code-Review\+2.*start-gate-scenario:.*'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: 'apex'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: 'ci/**'
+                  - compare-type: ANT
+                    pattern: 'build/**'
+                  - compare-type: ANT
+                    pattern: 'lib/**'
+                  - compare-type: ANT
+                    pattern: 'config/**'
+
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-daily.*'
+                - 'apex-deploy.*'
+                - 'apex-build.*'
+                - 'apex-runner.*'
+                - 'apex-verify.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=gate
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO={verify-scenario}
+                  FUNCTEST_SUITE_NAME=healthcheck
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: false
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+
+- job-template:
+    name: 'apex-runner-cperf-{stream}'
+
+    # runner cperf job
+    project-type: 'multijob'
+    node: 'intel-pod2'
+
+    disabled: false
+
+    parameters:
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: "Used for overriding the GIT URL coming from parameters macro."
+
+    scm:
+        - git-scm
+
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: false
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-deploy.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME"
+        - multijob:
+            name: 'Baremetal Deploy'
+            condition: ALWAYS
+            projects:
+                - name: 'apex-deploy-baremetal-{stream}'
+                  node-parameters: false
+                  current-parameters: true
+                  predefined-parameters: |
+                    OPNFV_CLEAN=yes
+                    DEPLOY_SCENARIO={verify-scenario}
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: Functest
+            condition: ALWAYS
+            projects:
+                - name: 'functest-apex-baremetal-daily-{stream}'
+                  node-parameters: true
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO={verify-scenario}
+                  kill-phase-on: NEVER
+                  abort-all-job: false
+                  git-revision: false
+
+# Build phase
+- job-template:
+    name: 'apex-build-{stream}'
+
+    # Job template for builds
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: '{build-slave}'
+
+    disabled: false
+
+    concurrent: true
+
+    parameters:
+        - '{project}-defaults'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+
+    scm:
+        - git-scm
+
+    properties:
+        - logrotate-default
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - 'apex-build'
+        - inject:
+           properties-content: ARTIFACT_TYPE=rpm
+        - 'apex-upload-artifact'
+
+# ISO verify job
+- job-template:
+    name: 'apex-verify-iso-{stream}'
+
+    # Job template for builds
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: '{virtual-slave}'
+
+    disabled: false
+
+    concurrent: true
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: "Used for overriding the GIT URL coming from parameters macro."
+
+    scm:
+        - git-scm
+
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-deploy.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - 'apex-iso-verify'
+        - inject:
+           properties-content: ARTIFACT_TYPE=iso
+        - 'apex-upload-artifact'
+
+# Deploy job
+- job-template:
+    name: 'apex-deploy-{platform}-{stream}'
+
+    # Job template for virtual deployment
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: 'apex-{platform}-{stream}'
+
+    concurrent: true
+
+    disabled: false
+
+    scm:
+        - git-scm
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - string:
+            name: DEPLOY_SCENARIO
+            default: '{verify-scenario}'
+            description: "Scenario to deploy with."
+        - string:
+            name: OPNFV_CLEAN
+            default: 'no'
+            description: "Use yes in lower case to invoke clean. Indicates if the deploy environment should be cleaned before deployment"
+
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-deploy.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+
+    builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
+        - 'apex-download-artifact'
+        - 'apex-deploy'
+        - 'apex-workspace-cleanup'
+
+
+# Baremetal Deploy and Test
+- job-template:
+    name: 'apex-deploy-test-baremetal-{stream}'
+
+    # Job template for daily build
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    project-type: 'multijob'
+
+    disabled: false
+
+    scm:
+        - git-scm
+
+    parameters:
+        - '{project}-defaults'
+        - '{project}-baremetal-{stream}-defaults'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - string:
+            name: DEPLOY_SCENARIO
+            default: '{verify-scenario}'
+            description: "Scenario to deploy with."
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-verify.*'
+                - 'apex-deploy.*'
+                - 'apex-runner.*'
+                - 'apex-.*-promote.*'
+    builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
+        - multijob:
+            name: 'Baremetal Deploy'
+            condition: ALWAYS
+            projects:
+                - name: 'apex-deploy-baremetal-{stream}'
+                  node-parameters: true
+                  current-parameters: true
+                  predefined-parameters: |
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: Functest
+            condition: ALWAYS
+            projects:
+                - name: 'functest-apex-baremetal-daily-{stream}'
+                  node-parameters: true
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                  kill-phase-on: NEVER
+                  abort-all-job: false
+                  git-revision: false
+        - multijob:
+            name: Yardstick
+            condition: ALWAYS
+            projects:
+                - name: 'yardstick-apex-baremetal-daily-{stream}'
+                  node-parameters: true
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                  kill-phase-on: NEVER
+                  abort-all-job: false
+                  git-revision: false
+
+{% for stream in scenarios %}
+# {{ stream }} Daily
+- job-template:
+    name: 'apex-daily-{{ stream }}'
+
+    # Job template for daily build
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    project-type: 'multijob'
+
+    node: '{build-slave}'
+
+    disabled: false
+
+    scm:
+        - git-scm
+
+    parameters:
+        - '{project}-defaults'
+        - '{project}-baremetal-{{ stream }}-defaults'
+        - project-parameter:
+            project: '{project}'
+            branch: '{{ branch[stream] }}'
+        - apex-parameter:
+            gs-pathname: '{{ gspathname[stream] }}'
+
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-verify.*'
+                - 'apex-deploy.*'
+                - 'apex-runner.*'
+                - 'apex-.*-promote.*'
+
+    triggers:
+        - 'apex-{{ stream }}'
+
+    builders:
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{{ stream }}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: 'Verify and upload ISO'
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-verify-iso-{{ stream }}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    BUILD_DIRECTORY=apex-build-{{ stream }}/.build
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: Baremetal Deploy and Test Phase
+            condition: SUCCESSFUL
+            projects:
+{% for scenario in scenarios[stream] %}
+                - name: 'apex-deploy-test-baremetal-{{ stream }}'
+                  node-parameters: false
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO={{scenario}}
+                    OPNFV_CLEAN=yes
+                  kill-phase-on: NEVER
+                  abort-all-job: true
+                  git-revision: false
+{% endfor %}
+{% endfor %}
+
+# snapshot create
+- job-template:
+    name: 'apex-create-snapshot'
+
+    # Job template for clean
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+
+    disabled: false
+
+    builders:
+        - shell:
+            !include-raw-escape: ./apex-snapshot-create.sh
+
+# snapshot upload
+- job-template:
+    name: 'apex-upload-snapshot'
+
+    # Job template for clean
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+
+    disabled: false
+
+    builders:
+        - inject:
+           properties-content: ARTIFACT_TYPE=snapshot
+        - 'apex-upload-artifact'
+
+# CSIT promote
+- job-template:
+    name: 'apex-csit-promote-daily-{stream}'
+
+    # Job template for promoting CSIT Snapshots
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: '{virtual-slave}'
+    project-type: 'multijob'
+    disabled: false
+
+    scm:
+        - git-scm
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+
+    properties:
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-verify.*'
+                - 'apex-deploy.*'
+                - 'apex-build.*'
+                - 'apex-runner.*'
+                - 'apex-daily.*'
+
+    triggers:
+        - timed: '0 12 * * 0'
+
+    builders:
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l3-csit-noha
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO=os-odl_l3-nofeature-noha
+                  FUNCTEST_SUITE_NAME=tempest_smoke_serial
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+        - multijob:
+            name: create snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+        - multijob:
+            name: upload snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+
+# FDIO promote
+- job-template:
+    name: 'apex-fdio-promote-daily-{stream}'
+
+    # Job template for promoting CSIT Snapshots
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: '{virtual-slave}'
+    project-type: 'multijob'
+    disabled: false
+
+    scm:
+        - git-scm
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+
+    properties:
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-verify.*'
+                - 'apex-deploy.*'
+                - 'apex-build.*'
+                - 'apex-runner.*'
+                - 'apex-daily.*'
+
+    builders:
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    DEPLOY_SCENARIO=os-odl_l2-netvirt_gbp_fdio-noha
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: false
+        - multijob:
+            name: create snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+        - multijob:
+            name: upload snapshot
+            condition: SUCCESSFUL
+            projects:
+              - name: 'apex-create-snapshot'
+                current-parameters: true
+                node-parameters: true
+                kill-phase-on: FAILURE
+                abort-all-job: true
+                git-revision: false
+
+- job-template:
+    name: 'apex-gs-clean-{stream}'
+
+    # Job template for clean
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    node: '{slave}'
+
+    disabled: false
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+
+    builders:
+        - 'apex-gs-clean'
+
+    triggers:
+        - 'apex-gs-clean-{stream}'
+
+########################
+# parameter macros
+########################
+- parameter:
+    name: apex-parameter
+    parameters:
+        - string:
+            name: ARTIFACT_NAME
+            default: 'latest'
+            description: "RPM Artifact name that will be appended to GS_URL to deploy a specific artifact"
+        - string:
+            name: ARTIFACT_VERSION
+            default: 'daily'
+            description: "Artifact version type"
+        - string:
+            name: BUILD_DIRECTORY
+            default: $WORKSPACE/.build
+            description: "Directory where the build artifact will be located upon the completion of the build."
+        - string:
+            name: CACHE_DIRECTORY
+            default: $HOME/opnfv/cache{gs-pathname}
+            description: "Directory where the cache to be used during the build is located."
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
+        - string:
+            name: GS_PATHNAME
+            default: '{gs-pathname}'
+            description: "Version directory where opnfv artifacts are stored in gs repository"
+        - string:
+            name: GS_URL
+            default: $GS_BASE{gs-pathname}
+            description: "URL to Google Storage."
+
+########################
+# builder macros
+########################
+- builder:
+    name: 'apex-unit-test'
+    builders:
+        - shell:
+            !include-raw: ./apex-unit-test.sh
+
+- builder:
+    name: 'apex-build'
+    builders:
+        - shell:
+            !include-raw: ./apex-build.sh
+
+- builder:
+    name: 'apex-workspace-cleanup'
+    builders:
+        - shell:
+            !include-raw: ./apex-workspace-cleanup.sh
+
+- builder:
+    name: 'apex-iso-verify'
+    builders:
+        - shell:
+            !include-raw: ./apex-iso-verify.sh
+
+
+- builder:
+    name: 'apex-upload-artifact'
+    builders:
+        - shell:
+            !include-raw: ./apex-upload-artifact.sh
+
+- builder:
+    name: 'apex-download-artifact'
+    builders:
+        - shell:
+            !include-raw: ./apex-download-artifact.sh
+
+- builder:
+    name: 'apex-gs-cleanup'
+    builders:
+        - shell:
+            !include-raw: ./apex-gs-cleanup.sh
+
+- builder:
+    name: 'apex-deploy'
+    builders:
+        - shell:
+            !include-raw: ./apex-deploy.sh
+
+#######################
+# trigger macros
+########################
+- trigger:
+    name: 'apex-master'
+    triggers:
+        - timed: '0 3 1 1 7'
+- trigger:
+    name: 'apex-danube'
+    triggers:
+        - timed: '0 12 * * *'
+- trigger:
+    name: 'apex-gs-clean-{stream}'
+    triggers:
+        - timed: '0 2 * * *'
diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden
new file mode 100644
index 000000000..bff7d3b5a
--- /dev/null
+++ b/jjb/apex/scenarios.yaml.hidden
@@ -0,0 +1,32 @@
+master:
+  - 'os-nosdn-nofeature-noha'
+  - 'os-nosdn-nofeature-ha'
+  - 'os-odl-nofeature-ha'
+  - 'os-odl-nofeature-noha'
+danube:
+  - 'os-nosdn-nofeature-noha'
+  - 'os-nosdn-nofeature-ha'
+  - 'os-nosdn-nofeature-ha-ipv6'
+  - 'os-nosdn-ovs-noha'
+  - 'os-nosdn-ovs-ha'
+  - 'os-nosdn-fdio-noha'
+  - 'os-nosdn-fdio-ha'
+  - 'os-nosdn-kvm-ha'
+  - 'os-nosdn-kvm-noha'
+  - 'os-odl_l2-fdio-noha'
+  - 'os-odl_l2-fdio-ha'
+  - 'os-odl_l2-netvirt_gbp_fdio-noha'
+  - 'os-odl_l2-sfc-noha'
+  - 'os-odl_l3-nofeature-noha'
+  - 'os-odl_l3-nofeature-ha'
+  - 'os-odl_l3-ovs-noha'
+  - 'os-odl_l3-ovs-ha'
+  - 'os-odl-bgpvpn-ha'
+  - 'os-odl-gluon-noha'
+  - 'os-odl_l3-fdio-noha'
+  - 'os-odl_l3-fdio-ha'
+  - 'os-odl_l3-fdio_dvr-noha'
+  - 'os-odl_l3-fdio_dvr-ha'
+  - 'os-odl_l3-csit-noha'
+  - 'os-onos-nofeature-ha'
+  - 'os-ovn-nofeature-noha'
diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml
index 3c04a4ac0..b180f591b 100644
--- a/jjb/functest/functest-daily-jobs.yml
+++ b/jjb/functest/functest-daily-jobs.yml
@@ -80,20 +80,20 @@
             installer: compass
             <<: *danube
 # apex CI PODs
-        - apex-verify-master:
-            slave-label: '{pod}'
+        - virtual:
+            slave-label: apex-virtual-master
             installer: apex
             <<: *master
-        - apex-daily-master:
-            slave-label: '{pod}'
+        - baremetal:
+            slave-label: apex-baremetal-master
             installer: apex
             <<: *master
-        - apex-verify-danube:
-            slave-label: '{pod}'
+        - virtual:
+            slave-label: apex-virtual-danube
             installer: apex
             <<: *danube
-        - apex-daily-danube:
-            slave-label: '{pod}'
+        - baremetal:
+            slave-label: apex-baremetal-danube
             installer: apex
             <<: *danube
 # armband CI PODs
diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml
index fad06b077..44666a7d1 100644
--- a/jjb/global/slave-params.yml
+++ b/jjb/global/slave-params.yml
@@ -4,11 +4,11 @@
 # use of the new labels are in place
 #####################################################
 - parameter:
-    name: 'apex-daily-master-defaults'
+    name: 'apex-baremetal-master-defaults'
     parameters:
         - label:
             name: SLAVE_LABEL
-            default: 'apex-daily-master'
+            default: 'apex-baremetal-master'
         - string:
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/$PROJECT
@@ -25,11 +25,11 @@
             default-slaves:
                 - lf-pod1
 - parameter:
-    name: 'apex-daily-danube-defaults'
+    name: 'apex-baremetal-danube-defaults'
     parameters:
         - label:
             name: SLAVE_LABEL
-            default: 'apex-daily-danube'
+            default: 'apex-baremetal-danube'
         - string:
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/$PROJECT
@@ -46,11 +46,11 @@
             default-slaves:
                 - lf-pod1
 - parameter:
-    name: 'apex-verify-master-defaults'
+    name: 'apex-virtual-master-defaults'
     parameters:
         - label:
             name: SLAVE_LABEL
-            default: 'apex-verify-master'
+            default: 'apex-virtual-master'
         - string:
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/$PROJECT
@@ -63,19 +63,18 @@
             name: SLAVE_NAME
             description: 'Slave name on Jenkins'
             allowed-slaves:
-                - intel-virtual3
-                - intel-virtual4
-                - intel-virtual5
+                - lf-virtual2
+                - lf-virtual3
             default-slaves:
-                - intel-virtual3
-                - intel-virtual4
-                - intel-virtual5
+                - lf-virtual2
+                - lf-virtual3
+
 - parameter:
-    name: 'apex-verify-danube-defaults'
+    name: 'apex-virtual-danube-defaults'
     parameters:
         - label:
             name: SLAVE_LABEL
-            default: 'apex-verify-danube'
+            default: 'apex-virtual-danube'
         - string:
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/$PROJECT
@@ -88,13 +87,9 @@
             name: SLAVE_NAME
             description: 'Slave name on Jenkins'
             allowed-slaves:
-                - intel-virtual3
-                - intel-virtual4
-                - intel-virtual5
+                - lf-pod3
             default-slaves:
-                - intel-virtual3
-                - intel-virtual4
-                - intel-virtual5
+                - lf-pod3
 - parameter:
     name: 'lf-pod1-defaults'
     parameters:
@@ -113,6 +108,24 @@
             name: SSH_KEY
             default: /root/.ssh/id_rsa
             description: 'SSH key to use for Apex'
+- parameter:
+    name: 'lf-pod3-defaults'
+    parameters:
+        - node:
+            name: SLAVE_NAME
+            description: 'Slave name on Jenkins'
+            allowed-slaves:
+                - lf-pod3
+            default-slaves:
+                - lf-pod3
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: 'Git URL to use on this Jenkins Slave'
+        - string:
+            name: SSH_KEY
+            default: /root/.ssh/id_rsa
+            description: 'SSH key to use for Apex'
 #####################################################
 # Parameters for CI baremetal PODs
 #####################################################