Adds Utility functions and failure logging

Patch adds helpful utility functions and automatic parsing of heat
output into a readable format indicating where a failure is in a
deployment.  Example: http://fpaste.org/354210/

New opnfv-util function allows a user to interact with a deployment
easily:
 - opnfv-util undercloud <user> connects user to undercloud VM
 - opnfv-util debug-stack parses the heat failure output

Above arguments also accept partial matches.

JIRA: APEX-75

Change-Id: I5ccfee64ee2958de0d00a3b25cd9b29de60c9e20
Signed-off-by: Tim Rozet <trozet@redhat.com>

4 files changed, 133 insertions, 1 deletions

diff --git a/build/opnfv-apex-common.spec b/build/opnfv-apex-common.spec
index 1ffac9d2..876755ca 100644
--- a/build/opnfv-apex-common.spec
+++ b/build/opnfv-apex-common.spec
@@ -28,6 +28,7 @@ rst2html docs/release-notes/release-notes.rst docs/release-notes.html
 mkdir -p %{buildroot}%{_bindir}/
 install ci/deploy.sh %{buildroot}%{_bindir}/opnfv-deploy
 install ci/clean.sh %{buildroot}%{_bindir}/opnfv-clean
+install ci/util.sh %{buildroot}%{_bindir}/opnfv-util
 
 mkdir -p %{buildroot}%{_sysconfdir}/opnfv-apex/
 install config/deploy/os-nosdn-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-ha.yaml
@@ -40,6 +41,7 @@ install config/network/network_settings.yaml %{buildroot}%{_sysconfdir}/opnfv-ap
 
 mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/
 install lib/common-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
+install lib/utility-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
 mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
 install lib/installer/onos/onos_gw_mac_update.sh %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
 
@@ -56,7 +58,9 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %defattr(644, root, root, -)
 %attr(755,root,root) %{_bindir}/opnfv-deploy
 %attr(755,root,root) %{_bindir}/opnfv-clean
+%attr(755,root,root) %{_bindir}/opnfv-util
 %{_var}/opt/opnfv/lib/common-functions.sh
+%{_var}/opt/opnfv/lib/utility-functions.sh
 %{_var}/opt/opnfv/lib/installer/onos/onos_gw_mac_update.sh
 %{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-odl_l2-nofeature-ha.yaml
diff --git a/ci/deploy.sh b/ci/deploy.sh
index c14a91e3..3684ad56 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -974,8 +974,12 @@ EOI
 
   ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
 source stackrc
-set -o errexit
 openstack overcloud deploy --templates $DEPLOY_OPTIONS --timeout 90
+if ! heat stack-list | grep CREATE_COMPLETE 1>/dev/null; then
+  $(typeset -f debug_stack)
+  debug_stack
+  exit 1
+fi
 EOI
 
   if [ "$debug" == 'TRUE' ]; then
diff --git a/ci/util.sh b/ci/util.sh
new file mode 100644
index 00000000..4f795456
--- /dev/null
+++ b/ci/util.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Utility script used to interact with a deployment
+# @author Tim Rozet (trozet@redhat.com)
+
+CONFIG=/var/opt/opnfv
+VALID_CMDS="undercloud debug-stack -h --help"
+
+source $CONFIG/utility-functions.sh
+
+resolve_cmd() {
+  local given=$1
+  shift
+  local list=($*)
+  local inv=(${list[*]##${given}*})
+  local OIFS=$IFS; IFS='|'; local pat="${inv[*]}"; IFS=$OIFS
+  shopt -s extglob
+  echo "${list[*]##+($pat)}"
+  shopt -u extglob
+}
+
+display_usage() {
+  echo -e "Usage:\n$0 [arguments] \n"
+  echo -e "   undercloud <user> : Connect to Undercloud VM as <user>\n"
+  echo -e "   debug_stack : Print parsed deployment failures to stdout \n"
+}
+
+##translates the command line argument
+##params: $@ the entire command line is passed
+##usage: parse_cmd_line() "$@"
+parse_cmdline() {
+  local match
+
+  match=($(resolve_cmd $1 $VALID_CMDS))
+  if [ ${#match[*]} -gt 1 ]; then
+    echo "$1 is ambiguous, possible matches: ${match[*]}" >&2
+    exit 1
+  elif [ ${#match[*]} -lt 1 ]; then
+    echo "$1 is not a recognized command.  Use -h to see acceptable list" >&2
+    exit 1
+  else
+    match=$(echo $match | tr -d ' ')
+  fi
+
+  case "$match" in
+        -h|--help)
+                display_usage
+                exit 0
+            ;;
+        undercloud)
+                if [ -z "$2" ]; then
+                  # connect as stack by default
+                  undercloud_connect stack
+                else
+                  undercloud_connect $2
+                fi
+                exit 0
+            ;;
+        debug-stack)
+                undercloud_connect stack "$(typeset -f debug_stack); debug_stack"
+                exit 0
+            ;;
+        *)
+                echo -e "\n\nThis script is used to interact with Apex deployments\n\n"
+                echo "Use -h to display help"
+                exit 1
+            ;;
+  esac
+}
+
+
+main() {
+  parse_cmdline "$@"
+}
+
+main "$@"
\ No newline at end of file
diff --git a/lib/utility-functions.sh b/lib/utility-functions.sh
new file mode 100644
index 00000000..17fdfafd
--- /dev/null
+++ b/lib/utility-functions.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Utility Functions used by  OPNFV Apex
+# author: Tim Rozet (trozet@redhat.com)
+
+##connects to undercloud
+##params: user to login with, command to execute on undercloud (optional)
+function undercloud_connect {
+  local user=$1
+
+  if [ -z "$1" ]; then
+    echo "Missing required argument: user to login as to undercloud"
+    return 1
+  fi
+
+  if [ -z "$2" ]; then
+    ssh ${user}@$(arp -a | grep $(virsh domiflist undercloud | grep default |\
+    awk '{print $5}') | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+")
+  else
+    ssh -T ${user}@$(arp -a | grep $(virsh domiflist undercloud | grep default \
+    | awk '{print $5}') | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+") "$2"
+  fi
+}
+
+##outputs heat stack deployment failures
+##params: none
+function debug_stack {
+  local failure_output
+  local phys_id
+  declare -a resource_arr
+  declare -a phys_id_arr
+
+  source ~/stackrc
+
+  IFS=$'\n'
+  for resource in $(heat resource-list -n 5 overcloud | grep FAILED); do
+    unset IFS
+    resource_arr=(${resource//|/ })
+    phys_id=$(heat resource-show ${resource_arr[-1]} ${resource_arr[0]} | grep physical_resource_id 2> /dev/null)
+    if [ -n "$phys_id" ]; then
+      phys_id_arr=(${phys_id//|/ })
+      failure_output+="******************************************************"
+      failure_output+="\n${resource}:\n\n$(heat deployment-show ${phys_id_arr[-1]} 2> /dev/null)"
+      failure_output+="\n******************************************************"
+    fi
+    unset phys_id
+  done
+
+  echo -e $failure_output
+}
\ No newline at end of file