[MaaS] Override failed testing by default

Some nodes fail automatic testing done by MaaS during commissioning,
although running the testing suites one more time manually works.
For now, just override all 'failed testing' nodes unconditionally.

JIRA: FUEL-333

Change-Id: I13d3ee3d82550524480aa53aa8752ab90aa940cd
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

4 files changed, 30 insertions, 1 deletions

diff --git a/mcp/config/states/maas b/mcp/config/states/maas
index 116bc45cd..0034e78f1 100755
--- a/mcp/config/states/maas
+++ b/mcp/config/states/maas
@@ -30,12 +30,19 @@ function maas_fixup() {
 
   local fcnodes=$(echo "${statusout}" | \
     grep -Pzo 'status: Failed commissioning\n\s+system_id: \K.+\n')
+  local ftnodes=$(echo "${statusout}" | \
+    grep -Pzo 'status: Failed testing\n\s+system_id: \K.+\n')
   for node_system_id in ${fcnodes}; do
     salt -C 'mas01*' state.apply maas.machines.delete \
       pillar="{'system_id': '${node_system_id}'}"
     sleep 10
   done
-  if [ -n "${fcnodes}" ]; then
+  for node_system_id in ${ftnodes}; do
+    salt -C 'mas01*' state.apply maas.machines.override_failed_testing \
+      pillar="{'system_id': '${node_system_id}'}"
+    sleep 10
+  done
+  if [ -n "${fcnodes}" ] || [ -n "${ftnodes}" ]; then
     salt -C 'mas01*' state.apply maas.machines
     return 1
   fi
diff --git a/mcp/salt-formulas/maas/machines/delete.sls b/mcp/salt-formulas/maas/machines/delete.sls
index dbb625a02..2903f9226 100644
--- a/mcp/salt-formulas/maas/machines/delete.sls
+++ b/mcp/salt-formulas/maas/machines/delete.sls
@@ -10,6 +10,7 @@
 maas_login_admin:
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
 
 # TODO: implement delete_machine via _modules/maas.py
 delete_machine:
diff --git a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
index 17a7df8d8..3983729e7 100644
--- a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
+++ b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
@@ -10,6 +10,7 @@
 maas_login_admin:
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
 
 # TODO: implement mark_broken_fixed_machine via _modules/maas.py
 mark_broken_fixed_machine:
diff --git a/mcp/salt-formulas/maas/machines/override_failed_testing.sls b/mcp/salt-formulas/maas/machines/override_failed_testing.sls
new file mode 100644
index 000000000..e7fe1d267
--- /dev/null
+++ b/mcp/salt-formulas/maas/machines/override_failed_testing.sls
@@ -0,0 +1,20 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+{%- from "maas/map.jinja" import region with context %}
+
+maas_login_admin:
+  cmd.run:
+  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
+
+# TODO: implement override_failed_testing via _modules/maas.py
+mark_broken_fixed_machine:
+  cmd.run:
+  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}"
+  - require:
+    - cmd: maas_login_admin