states: Break on error, retry states up to 5 times

While applying scenario states, break on error, and retry failed
state up to 5 times. Apply the same behavior for `salt.sh`.

Add new deploy parameter, '-D', backed up by 'CI_DEBUG' env var,
which gates deploy sh scripts logging (set -x).
Also extend '-f' deploy parameter, allowing it to be specified
more than once; the first occurence will skip infra VM creation,
but still sync reclass & other config from local repo, while a
second occurence will also disable config sync.

To prevent glusterfs client state from failing due to non-existent
nova user/group, move it after nova:compute's nova state is applied.

Change-Id: I234e126e16be0e133d878957bd88fed946955de8
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
(cherry picked from commit 23f9ef727310a8ef449372313785241df6557559)

8 files changed, 26 insertions, 10 deletions

diff --git a/mcp/config/states/dpdk b/mcp/config/states/dpdk
index 9ae103d90..eb00d7279 100755
--- a/mcp/config/states/dpdk
+++ b/mcp/config/states/dpdk
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 # shellcheck disable=SC1090
 source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
 
diff --git a/mcp/config/states/maas b/mcp/config/states/maas
index 685dbc9bd..d6ef5a607 100755
--- a/mcp/config/states/maas
+++ b/mcp/config/states/maas
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 # shellcheck disable=SC1090
 source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
 
diff --git a/mcp/config/states/networks b/mcp/config/states/networks
index 361e675a8..d4735adb3 100755
--- a/mcp/config/states/networks
+++ b/mcp/config/states/networks
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 # Determine public network based on external IPs from compute node
 # NOTE: mask currently hardcoded to /24
 PUBLIC_NET=$(salt --out yaml 'cmp*' pillar.get _param:external_address | \
diff --git a/mcp/config/states/neutron_compute b/mcp/config/states/neutron_compute
index 2a2a2ef78..70a2db607 100755
--- a/mcp/config/states/neutron_compute
+++ b/mcp/config/states/neutron_compute
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 salt -I 'neutron:compute' state.sls neutron
 salt -I 'neutron:compute' file.append /etc/sudoers.d/neutron_sudoers \
   args='neutron ALL = (root) NOPASSWD: /usr/bin/neutron-rootwrap-daemon /etc/neutron/rootwrap.conf'
diff --git a/mcp/config/states/opendaylight b/mcp/config/states/opendaylight
index 7b63483d8..34a2c3bd8 100755
--- a/mcp/config/states/opendaylight
+++ b/mcp/config/states/opendaylight
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 salt -I 'opendaylight:server' state.sls opendaylight
 salt -I 'opendaylight:client' state.sls opendaylight.client
 salt -I 'opendaylight:client' cmd.run 'neutron-odl-ovs-hostconfig --noovs_dpdk'
diff --git a/mcp/config/states/openstack b/mcp/config/states/openstack
index 968310334..19cd41703 100755
--- a/mcp/config/states/openstack
+++ b/mcp/config/states/openstack
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 salt -I 'nfs:server' file.mkdir /srv/nova/instances
 salt -I 'nfs:server' state.sls nfs
 salt -I 'nfs:client' state.sls nfs
diff --git a/mcp/config/states/openstack_ha b/mcp/config/states/openstack_ha
index 230288f4f..2e55fc890 100755
--- a/mcp/config/states/openstack_ha
+++ b/mcp/config/states/openstack_ha
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 # shellcheck disable=SC1090
 source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
 
@@ -37,7 +39,6 @@ salt -I 'keystone:client' state.sls keystone.client
 salt -I 'keystone:server' cmd.run ". /root/keystonercv3; openstack service list"
 
 salt -I 'glance:server' state.sls glance -b 1
-salt -I 'glusterfs:client' state.sls glusterfs.client
 salt -I 'keystone:server' state.sls keystone.server
 
 salt -I 'nova:controller' state.sls nova -b 1
@@ -50,6 +51,7 @@ salt -I 'neutron:server' state.sls neutron -b 1
 salt -I 'neutron:gateway' state.sls neutron.gateway
 
 salt -I 'nova:compute' state.sls nova
+salt -I 'glusterfs:client' state.sls glusterfs.client
 
 salt -I 'mongodb:server' state.sls mongodb
 wait_for 90 "salt -C 'I@mongodb:server and *01*' cmd.run 'mongo localhost:27017/admin'"
diff --git a/mcp/config/states/virtual_control_plane b/mcp/config/states/virtual_control_plane
index 58ae33506..a0e3a31d0 100755
--- a/mcp/config/states/virtual_control_plane
+++ b/mcp/config/states/virtual_control_plane
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -e
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
 # shellcheck disable=SC1090
 source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
 
@@ -18,7 +20,7 @@ wait_for 90 "! salt 'kvm*' test.ping | tee /dev/stderr | grep -Fq 'Not connected
 
 salt -C '* and not cfg01* and not mas01*' state.apply linux,ntp
 
-salt -C 'kvm*' state.sls libvirt
+wait_for 5 "salt -C 'kvm*' state.sls libvirt"
 
 salt -C '* and not cfg01* and not mas01*' state.apply salt
 salt -C 'kvm*' saltutil.sync_all