Apex: Create promotion and verify job for Apex CSIT snapshots

Changes Include:
 - Adds a daily job to create Apex snapshot tarball if functest passes
 - snapshot artifact uploaded to artifacts.opnfv.org
 - snapshot artifcat info is stored in snapshot.properties
 - 3rd party job updated to deploy from snapshot
 - 3rd party install netvirt job updated to use updated sdnvpn tools

JIRA: APEX-364

Depends-On: I792db0fac3f4e81969fe85c05fc298fe5af02537

Change-Id: I8a875c081d756d098173c0baf11c0b4e4956b9fd
Signed-off-by: Tim Rozet <trozet@redhat.com>
Signed-off-by: Nikolas Hermanns <nikolas.hermanns@ericsson.com>
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

7 files changed, 361 insertions, 20 deletions

diff --git a/jjb/3rd_party_ci/install-netvirt.sh b/jjb/3rd_party_ci/install-netvirt.sh
index f111d4847..c9aa4c501 100755
--- a/jjb/3rd_party_ci/install-netvirt.sh
+++ b/jjb/3rd_party_ci/install-netvirt.sh
@@ -2,14 +2,28 @@
 set -e
 
 if [ -z ${WORKSPACE} ]; then
-  echo "WORKSPACE is unset. Please do so."
+  echo "WORKSPACE is unset. Please set."
   exit 1
 fi
 # wipe the WORKSPACE
 /bin/rm -rf $WORKSPACE/*
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SNAP_CACHE=$HOME/snap_cache
 # clone opnfv sdnvpn repo
 git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn
-. $WORKSPACE/sdnvpn/odl-pipeline/odl-pipeline-common.sh
-pushd $LIB
-./odl_reinstaller.sh --cloner-info $CLONER_INFO --odl-artifact $NETVIRT_ARTIFACT
-popd
\ No newline at end of file
+
+if [ ! -f "$NETVIRT_ARTIFACT" ]; then
+  echo "ERROR: ${NETVIRT_ARTIFACT} specified as NetVirt Artifact, but file does not exist"
+  exit 1
+fi
+
+# TODO (trozet) snapshot should have already been unpacked into cache folder
+# but we really should check the cache here, and not use a single cache folder
+# for when we support multiple jobs on a single slave
+pushd sdnvpn/odl-pipeline/lib > /dev/null
+./odl_reinstaller.sh --pod-config ${SNAP_CACHE}/node.yaml \
+  --odl-artifact ${NETVIRT_ARTIFACT} --ssh-key-file ${SNAP_CACHE}/id_rsa
+popd > /dev/null
diff --git a/jjb/3rd_party_ci/odl-netvirt.yml b/jjb/3rd_party_ci/odl-netvirt.yml
index 0e479cc08..184a4980c 100644
--- a/jjb/3rd_party_ci/odl-netvirt.yml
+++ b/jjb/3rd_party_ci/odl-netvirt.yml
@@ -17,11 +17,11 @@
 #####################################
     phase:
         - 'create-apex-vms':
-            slave-label: 'ericsson-virtual5'
+            slave-label: 'odl-netvirt-virtual-intel'
         - 'install-netvirt':
-            slave-label: 'odl-netvirt-virtual'
+            slave-label: 'odl-netvirt-virtual-intel'
         - 'postprocess':
-            slave-label: 'odl-netvirt-virtual'
+            slave-label: 'odl-netvirt-virtual-intel'
 #####################################
 # jobs
 #####################################
@@ -62,10 +62,10 @@
         - gerrit:
             server-name: 'git.opendaylight.org'
             trigger-on:
-                - comment-added-contains-event:
-                    comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : SUCCESS'
-                - comment-added-contains-event:
-                    comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : UNSTABLE'
+ #               - comment-added-contains-event:
+ #                   comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : SUCCESS'
+ #               - comment-added-contains-event:
+ #                   comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : UNSTABLE'
                 - comment-added-contains-event:
                     comment-contains-value: 'opnfv-test'
             projects:
@@ -118,10 +118,10 @@
             name: functest
             condition: SUCCESSFUL
             projects:
-                - name: 'functest-netvirt-virtual-daily-{stream}'
+                - name: 'functest-netvirt-virtual-suite-{stream}'
                   predefined-parameters: |
-                    RC_FILE_PATH=/home/jenkins/cloner-info/overcloudrc
-                    DEPLOY_SCENARIO=os-odl_l2-bgpvpn-noha
+                    DEPLOY_SCENARIO=os-odl_l3-nofeature-ha
+                    FUNCTEST_SUITE_NAME=healthcheck
                   node-parameters: true
                   kill-phase-on: FAILURE
                   abort-all-job: false
@@ -141,7 +141,7 @@
                     NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT
                   node-parameters: true
                   kill-phase-on: FAILURE
-                  abort-all-job: true
+                  abort-all-job: false
 
 - job-template:
     name: 'odl-netvirt-verify-virtual-{phase}-{stream}'
@@ -160,8 +160,9 @@
         - build-blocker:
             use-build-blocker: true
             blocking-jobs:
-                - 'odl-netvirt-verify-virtual-install-.*'
-                - 'functest-netvirt-virtual-daily-.*'
+                - 'odl-netvirt-verify-virtual-create-apex-vms-.*'
+                - 'odl-netvirt-verify-virtual-install-netvirt-.*'
+                - 'functest-netvirt-virtual-suite-.*'
                 - 'odl-netvirt-verify-virtual-postprocess-.*'
             block-level: 'NODE'
 
@@ -190,7 +191,7 @@
     name: 'netvirt-verify-create-apex-vms-builder'
     builders:
         - shell:
-            !include-raw: ./create-apex-vms.sh
+            !include-raw: ../apex/apex-snapshot-deploy.sh
 - builder:
     name: 'netvirt-verify-install-netvirt-builder'
     builders:
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh
index 8d5c4cb13..776b76ed0 100755
--- a/jjb/apex/apex-deploy.sh
+++ b/jjb/apex/apex-deploy.sh
@@ -172,6 +172,9 @@ if [[ "$JOB_NAME" == *virtual* ]]; then
     NETWORK_FILE="${NETWORK_SETTINGS_DIR}/network_settings.yaml"
   fi
   DEPLOY_CMD="${DEPLOY_CMD} -v"
+  if [[ "$JOB_NAME" == *csit* ]]; then
+    DEPLOY_CMD="${DEPLOY_CMD} -e csit-environment.yaml --virtual-computes 2"
+  fi
 else
   # settings for bare metal deployment
   if [ "$IPV6_FLAG" == "True" ]; then
diff --git a/jjb/apex/apex-snapshot-create.sh b/jjb/apex/apex-snapshot-create.sh
new file mode 100644
index 000000000..09c6a1197
--- /dev/null
+++ b/jjb/apex/apex-snapshot-create.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (Red Hat) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
+
+echo "Creating Apex snapshot..."
+echo "-------------------------"
+echo
+
+# create tmp directory
+tmp_dir=$(pwd)/.tmp
+mkdir -p ${tmp_dir}
+
+# TODO(trozet) remove this after fix goes in for tripleo_inspector to copy these
+pushd ${tmp_dir} > /dev/null
+echo "Copying overcloudrc and ssh key from Undercloud..."
+# Store overcloudrc
+UNDERCLOUD=$(sudo virsh domifaddr undercloud | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]')
+scp ${SSH_OPTIONS[@]} stack@${UNDERCLOUD}:overcloudrc ./
+# Copy out ssh key of stack from undercloud
+scp ${SSH_OPTIONS[@]} stack@${UNDERCLOUD}:.ssh/id_rsa ./
+popd > /dev/null
+
+echo "Gathering introspection information"
+git clone https://gerrit.opnfv.org/gerrit/sdnvpn.git
+pushd sdnvpn/odl-pipeline/lib > /dev/null
+./tripleo_introspector.sh --out-file ${tmp_dir}/node.yaml
+popd > /dev/null
+
+echo "Shutting down nodes"
+# Shut down nodes
+nodes=$(sudo virsh list | grep -Eo "baremetal[0-9]")
+for node in $nodes; do
+  sudo virsh shutdown ${node} --mode acpi
+done
+
+for node in $nodes; do
+  count=0
+  while [ "$count" -lt 10 ]; do
+    sleep 10
+    if sudo virsh list | grep ${node}; then
+       echo "Waiting for $node to shutdown, try $count"
+    else
+       break
+    fi
+    count=$((count+1))
+  done
+
+  if [ "$count" -ge 10 ]; then
+    echo "Node $node failed to shutdown"
+    exit 1
+  fi
+done
+
+echo "Gathering virsh definitions"
+# copy qcow2s, virsh definitions
+for node in $nodes; do
+  cp -f /var/lib/libvirt/images/${node}.qcow2 ./
+  sudo virsh dumpxml ${node} > ${node}.xml
+done
+
+# copy virsh net definitions
+for net in admin api external storage tenant; do
+  sudo virsh net-dumpxml ${net} > ${net}.xml
+done
+
+# tar up artifacts
+DATE=`date +%Y-%m-%d`
+tar czf ../apex-csit-snap-${DATE}.tar.gz .
+popd > /dev/null
+rm -rf ./.tmp
+echo "Snapshot saved as apex-csit-snap-${DATE}.tar.gz"
+
+# update opnfv properties file
+curl -O -L http://$GS_URL/snapshot.properties
+sed -i '/^OPNFV_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#};${x;/^$/{s##OPNFV_SNAP_URL='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties
+snap_sha=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1)
+sed -i '/^OPNFV_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//OPNFV_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties
+echo "OPNFV_SNAP_URL=$GS_URL/apex-csit-snap-${DATE}.tar.gz"
+echo "OPNFV_SNAP_SHA512SUM=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1)"
+echo "Updated properties file: "
+cat snapshot.properties
diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh
new file mode 100644
index 000000000..3bb65a0b3
--- /dev/null
+++ b/jjb/apex/apex-snapshot-deploy.sh
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (Red Hat) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
+SNAP_CACHE=$HOME/snap_cache
+
+
+echo "Deploying Apex snapshot..."
+echo "--------------------------"
+echo
+
+echo "Cleaning server"
+git clone https://gerrit.opnfv.org/gerrit/apex.git
+pushd apex/ci > /dev/null
+sudo CONFIG=../build/ LIB=../lib ./clean.sh
+popd > /dev/null
+
+echo "Downloading latest snapshot properties file"
+if ! wget -O $WORKSPACE/opnfv.properties http://$GS_URL/snapshot.properties; then
+  echo "ERROR: Unable to find snapshot.properties at ${GS_URL}...exiting"
+  exit 1
+fi
+
+# find latest check sum
+latest_snap_checksum=$(cat opnfv.properties | grep OPNFV_SNAP_SHA512SUM | awk -F "=" '{print $2}')
+if [ -z "$latest_snap_checksum" ]; then
+  echo "ERROR: checksum of latest snapshot from snapshot.properties is null!"
+  exit 1
+fi
+
+local_snap_checksum=""
+
+# check snap cache directory exists
+if [ -d "$SNAP_CACHE" ]; then
+  latest_snap=$(ls -Art | grep tar.gz | tail -n 1)
+  if [ -n "$latest_snap" ]; then
+    local_snap_checksum=$(sha512sum ${latest_snap} | cut -d' ' -f1)
+  fi
+else
+  mkdir -p ${SNAP_CACHE}
+fi
+
+# compare check sum and download latest snap if not up to date
+if [ "$local_snap_checksum" -ne "$latest_snap_checksum" ]; then
+  snap_url=$(cat opnfv.properties | grep OPNFV_SNAP_URL | awk -F "=" '{print $2}')
+  if [ -z "$snap_url" ]; then
+    echo "ERROR: Snap URL from snapshot.properties is null!"
+    exit 1
+  fi
+  echo "INFO: SHA mismatch, will download latest snapshot"
+  wget --directory-prefix=${SNAP_CACHE}/ ${snap_url}
+  snap_tar=$(basename ${snap_url})
+else
+  snap_tar=${latest_snap}
+fi
+
+echo "INFO: Snapshot to be used is ${snap_tar}"
+
+# create tmp directory and unpack snap
+mkdir -p ./tmp
+pushd ./tmp > /dev/null
+tar xvf ${snap_tar}
+
+# create each network
+virsh_networks=$(ls *.xml | grep -v baremetal)
+
+if [ -z "$virsh_networks" ]; then
+  echo "ERROR: no virsh networks found in snapshot unpack"
+  exit 1
+fi
+
+for network_def in ${virsh_networks}; do
+  sudo virsh net-create ${network_def}
+  network=$(echo ${network_def} | awk -F '.' '{print $1}')
+  if ! sudo virsh net-list | grep ${network}; then
+    sudo virsh net-start ${network}
+  fi
+  echo "Checking if OVS bridge is missing for network: ${network}"
+  if ! ovs-vsctl show | grep "br-${network}"; then
+    ovs-vsctl add-br br-${network}
+    echo "OVS Bridge created: br-${network}"
+    if [ "br-${network}" == 'br-admin' ]; then
+      echo "Configuring IP 192.0.2.99 on br-admin"
+      sudo ip addr add  192.0.2.99/24 dev br-admin
+      sudo ip link set up dev br-admin
+    elif [ "br-${network}" == 'br-external' ]; then
+      echo "Configuring IP 192.168.37.99 on br-external"
+      sudo ip addr add  192.168.37.99/24 dev br-external
+      sudo ip link set up dev br-external
+    fi
+  fi
+done
+
+echo "Virsh networks up: $(virsh net-list)"
+echo "Bringing up Overcloud VMs..."
+virsh_vm_defs=$(ls baremetal*.xml)
+
+if [ -z "$virsh_vm_defs" ]; then
+  echo "ERROR: no virsh VMs found in snapshot unpack"
+  exit 1
+fi
+
+for node_def in ${virsh_vm_defs}; do
+  sudo virsh define ${node_def}
+  node=$(echo ${node_def} | awk -F '.' '{print $1}')
+  sudo cp -f ${node}.qcow2 /var/lib/libvirt/images/
+  sudo virsh start ${node}
+  echo "Node: ${node} started"
+done
+
+echo "Checking overcloudrc"
+if ! stat overcloudrc; then
+  echo "ERROR: overcloudrc does not exist in snap unpack"
+  exit 1
+fi
+
+# copy overcloudrc for functest
+mkdir -p $HOME/cloner-info
+cp -f overcloudrc $HOME/cloner-info/
+
+admin_controller_ip=$(cat overcloudrc | grep -Eo "192.0.2.[0-9]+")
+netvirt_url="http://${admin_controller_ip}:8081/restconf/operational/network-topology:network-topology/topology/netvirt:1"
+
+source overcloudrc
+counter=1
+while [ "$counter" -le 10 ]; do
+  if curl --fail ${admin_controller_ip}:80; then
+    echo "Overcloud Horizon is up...Checking if OpenDaylight NetVirt is up..."
+    if curl --fail ${netvirt_url} > /dev/null; then
+      echo "OpenDaylight is up.  Overcloud deployment complete"
+      exit 0
+    else
+      echo "OpenDaylight not yet up, try ${counter}"
+    fi
+  else
+    echo "Horizon/Apache not yet up, try ${counter}"
+  fi
+  counter=$((counter+1))
+  sleep 60
+done
+
+echo "ERROR: Deployment not up after 10 minutes...exiting."
+exit 1
diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh
index 64f13f4e6..89fd5ed36 100755
--- a/jjb/apex/apex-upload-artifact.sh
+++ b/jjb/apex/apex-upload-artifact.sh
@@ -73,7 +73,17 @@ gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION
 gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log
 }
 
-if gpg2 --list-keys | grep "opnfv-helpdesk@rt.linuxfoundation.org"; then
+uploadsnap () {
+  # Uploads snapshot artifact and updated properties file
+  echo "Uploading snapshot artifacts"
+  gsutil cp $WORKSPACE/apex-csit-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log
+  gsutil cp $WORKSPACE/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log
+  echo "Upload complete for Snapshot"
+}
+
+if grep csit $WORKSPACE; then
+  uploadsnap
+elif gpg2 --list-keys | grep "opnfv-helpdesk@rt.linuxfoundation.org"; then
   echo "Signing Key avaliable"
   signiso
   uploadiso
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml
index 710a5f42c..d028787cc 100644
--- a/jjb/apex/apex.yml
+++ b/jjb/apex/apex.yml
@@ -41,6 +41,7 @@
          - 'os-odl_l3-fdio-ha'
          - 'os-odl_l3-fdio_dvr-noha'
          - 'os-odl_l3-fdio_dvr-ha'
+         - 'os-odl_l3-csit-noha'
          - 'os-onos-nofeature-ha'
 
     platform:
@@ -721,6 +722,71 @@
                 failure-threshold: 'never'
                 unstable-threshold: 'FAILURE'
 
+# CSIT promote
+- job-template:
+    name: 'apex-csit-promote-daily-{stream}'
+
+    # Job template for promoting CSIT Snapshots
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: '{daily-slave}'
+
+    disabled: false
+
+    scm:
+        - git-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            branch: '{branch}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+
+    properties:
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-verify.*'
+                - 'apex-deploy.*'
+                - 'apex-build.*'
+                - 'apex-runner.*'
+
+    triggers:
+        - timed: '0 12 * * 0'
+
+    builders:
+        - trigger-builds:
+          - project: 'apex-build-{stream}'
+            git-revision: true
+            current-parameters: true
+            same-node: true
+            block: true
+        - trigger-builds:
+          - project: 'apex-deploy-virtual-os-odl_l3-csit-noha-{stream}'
+            predefined-parameters: |
+              BUILD_DIRECTORY=apex-build-{stream}/build
+              OPNFV_CLEAN=yes
+            git-revision: false
+            block: true
+            same-node: true
+        - trigger-builds:
+          - project: 'functest-apex-{verify-slave}-suite-{stream}'
+            predefined-parameters: |
+              DEPLOY_SCENARIO=os-odl_l3-nofeature-noha
+              FUNCTEST_SUITE_NAME=smoke
+            block: true
+            same-node: true
+        - shell:
+            !include-raw-escape: ./apex-snapshot-create.sh
+        - shell:
+            !include-raw-escape: ./apex-upload-artifact.sh
+
 - job-template:
     name: 'apex-gs-clean-{stream}'