summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTim Rozet <trozet@redhat.com>2016-04-11 14:22:27 -0400
committerTim Rozet <trozet@redhat.com>2016-04-12 14:31:50 -0400
commit0717bc3c56ab294f860ef99116754c1fd786cae7 (patch)
treedb98b4d669dab185fc3d6e5a668dbef19002393b
parentf7def4c0913df374087da447548aa6741bc6c8d7 (diff)
Adds Utility functions and failure logging
Patch adds helpful utility functions and automatic parsing of heat output into a readable format indicating where a failure is in a deployment. Example: http://fpaste.org/354210/ New opnfv-util function allows a user to interact with a deployment easily: - opnfv-util undercloud <user> connects user to undercloud VM - opnfv-util debug-stack parses the heat failure output Above arguments also accept partial matches. JIRA: APEX-75 Change-Id: I5ccfee64ee2958de0d00a3b25cd9b29de60c9e20 Signed-off-by: Tim Rozet <trozet@redhat.com>
-rw-r--r--build/opnfv-apex-common.spec4
-rwxr-xr-xci/deploy.sh6
-rw-r--r--ci/util.sh75
-rw-r--r--lib/utility-functions.sh49
4 files changed, 133 insertions, 1 deletions
diff --git a/build/opnfv-apex-common.spec b/build/opnfv-apex-common.spec
index 1ffac9d2..876755ca 100644
--- a/build/opnfv-apex-common.spec
+++ b/build/opnfv-apex-common.spec
@@ -28,6 +28,7 @@ rst2html docs/release-notes/release-notes.rst docs/release-notes.html
mkdir -p %{buildroot}%{_bindir}/
install ci/deploy.sh %{buildroot}%{_bindir}/opnfv-deploy
install ci/clean.sh %{buildroot}%{_bindir}/opnfv-clean
+install ci/util.sh %{buildroot}%{_bindir}/opnfv-util
mkdir -p %{buildroot}%{_sysconfdir}/opnfv-apex/
install config/deploy/os-nosdn-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-ha.yaml
@@ -40,6 +41,7 @@ install config/network/network_settings.yaml %{buildroot}%{_sysconfdir}/opnfv-ap
mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/
install lib/common-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
+install lib/utility-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
install lib/installer/onos/onos_gw_mac_update.sh %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
@@ -56,7 +58,9 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
%defattr(644, root, root, -)
%attr(755,root,root) %{_bindir}/opnfv-deploy
%attr(755,root,root) %{_bindir}/opnfv-clean
+%attr(755,root,root) %{_bindir}/opnfv-util
%{_var}/opt/opnfv/lib/common-functions.sh
+%{_var}/opt/opnfv/lib/utility-functions.sh
%{_var}/opt/opnfv/lib/installer/onos/onos_gw_mac_update.sh
%{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-ha.yaml
%{_sysconfdir}/opnfv-apex/os-odl_l2-nofeature-ha.yaml
diff --git a/ci/deploy.sh b/ci/deploy.sh
index c14a91e3..3684ad56 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -974,8 +974,12 @@ EOI
ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
source stackrc
-set -o errexit
openstack overcloud deploy --templates $DEPLOY_OPTIONS --timeout 90
+if ! heat stack-list | grep CREATE_COMPLETE 1>/dev/null; then
+ $(typeset -f debug_stack)
+ debug_stack
+ exit 1
+fi
EOI
if [ "$debug" == 'TRUE' ]; then
diff --git a/ci/util.sh b/ci/util.sh
new file mode 100644
index 00000000..4f795456
--- /dev/null
+++ b/ci/util.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Utility script used to interact with a deployment
+# @author Tim Rozet (trozet@redhat.com)
+
+CONFIG=/var/opt/opnfv
+VALID_CMDS="undercloud debug-stack -h --help"
+
+source $CONFIG/utility-functions.sh
+
+resolve_cmd() {
+ local given=$1
+ shift
+ local list=($*)
+ local inv=(${list[*]##${given}*})
+ local OIFS=$IFS; IFS='|'; local pat="${inv[*]}"; IFS=$OIFS
+ shopt -s extglob
+ echo "${list[*]##+($pat)}"
+ shopt -u extglob
+}
+
+display_usage() {
+ echo -e "Usage:\n$0 [arguments] \n"
+ echo -e " undercloud <user> : Connect to Undercloud VM as <user>\n"
+ echo -e " debug_stack : Print parsed deployment failures to stdout \n"
+}
+
+##translates the command line argument
+##params: $@ the entire command line is passed
+##usage: parse_cmd_line() "$@"
+parse_cmdline() {
+ local match
+
+ match=($(resolve_cmd $1 $VALID_CMDS))
+ if [ ${#match[*]} -gt 1 ]; then
+ echo "$1 is ambiguous, possible matches: ${match[*]}" >&2
+ exit 1
+ elif [ ${#match[*]} -lt 1 ]; then
+ echo "$1 is not a recognized command. Use -h to see acceptable list" >&2
+ exit 1
+ else
+ match=$(echo $match | tr -d ' ')
+ fi
+
+ case "$match" in
+ -h|--help)
+ display_usage
+ exit 0
+ ;;
+ undercloud)
+ if [ -z "$2" ]; then
+ # connect as stack by default
+ undercloud_connect stack
+ else
+ undercloud_connect $2
+ fi
+ exit 0
+ ;;
+ debug-stack)
+ undercloud_connect stack "$(typeset -f debug_stack); debug_stack"
+ exit 0
+ ;;
+ *)
+ echo -e "\n\nThis script is used to interact with Apex deployments\n\n"
+ echo "Use -h to display help"
+ exit 1
+ ;;
+ esac
+}
+
+
+main() {
+ parse_cmdline "$@"
+}
+
+main "$@" \ No newline at end of file
diff --git a/lib/utility-functions.sh b/lib/utility-functions.sh
new file mode 100644
index 00000000..17fdfafd
--- /dev/null
+++ b/lib/utility-functions.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Utility Functions used by OPNFV Apex
+# author: Tim Rozet (trozet@redhat.com)
+
+##connects to undercloud
+##params: user to login with, command to execute on undercloud (optional)
+function undercloud_connect {
+ local user=$1
+
+ if [ -z "$1" ]; then
+ echo "Missing required argument: user to login as to undercloud"
+ return 1
+ fi
+
+ if [ -z "$2" ]; then
+ ssh ${user}@$(arp -a | grep $(virsh domiflist undercloud | grep default |\
+ awk '{print $5}') | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+")
+ else
+ ssh -T ${user}@$(arp -a | grep $(virsh domiflist undercloud | grep default \
+ | awk '{print $5}') | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+") "$2"
+ fi
+}
+
+##outputs heat stack deployment failures
+##params: none
+function debug_stack {
+ local failure_output
+ local phys_id
+ declare -a resource_arr
+ declare -a phys_id_arr
+
+ source ~/stackrc
+
+ IFS=$'\n'
+ for resource in $(heat resource-list -n 5 overcloud | grep FAILED); do
+ unset IFS
+ resource_arr=(${resource//|/ })
+ phys_id=$(heat resource-show ${resource_arr[-1]} ${resource_arr[0]} | grep physical_resource_id 2> /dev/null)
+ if [ -n "$phys_id" ]; then
+ phys_id_arr=(${phys_id//|/ })
+ failure_output+="******************************************************"
+ failure_output+="\n${resource}:\n\n$(heat deployment-show ${phys_id_arr[-1]} 2> /dev/null)"
+ failure_output+="\n******************************************************"
+ fi
+ unset phys_id
+ done
+
+ echo -e $failure_output
+} \ No newline at end of file