[MaaS] Override failed testing by default

Some nodes fail automatic testing done by MaaS during commissioning,
although running the testing suites one more time manually works.
For now, just override all 'failed testing' nodes unconditionally.

[stable/euphrates cherry-pick additions]
Note: Since our salt formulas are pinned to 2017.12 repos, we
need to backport one salt-formula-maas patch merged upstream, which
adds support for translating status code '22' to 'Failed testing' [1].

JIRA: FUEL-333

[1] https://github.com/salt-formulas/salt-formula-maas/commit/08ffc3ff

Change-Id: I13d3ee3d82550524480aa53aa8752ab90aa940cd
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
(cherry picked from commit 81561126307f15d4f65a743ed2431ea8c713a921)

6 files changed, 69 insertions, 1 deletions

diff --git a/mcp/config/states/maas b/mcp/config/states/maas
index 39f6badef..24249d40d 100755
--- a/mcp/config/states/maas
+++ b/mcp/config/states/maas
@@ -27,12 +27,19 @@ function maas_fixup() {
 
   local fcnodes=$(echo "${statusout}" | \
     grep -Pzo 'status: Failed commissioning\n\s+system_id: \K.+\n')
+  local ftnodes=$(echo "${statusout}" | \
+    grep -Pzo 'status: Failed testing\n\s+system_id: \K.+\n')
   for node_system_id in ${fcnodes}; do
     salt -C 'mas01*' state.apply maas.machines.delete \
       pillar="{'system_id': '${node_system_id}'}"
     sleep 10
   done
-  if [ -n "${fcnodes}" ]; then
+  for node_system_id in ${ftnodes}; do
+    salt -C 'mas01*' state.apply maas.machines.override_failed_testing \
+      pillar="{'system_id': '${node_system_id}'}"
+    sleep 10
+  done
+  if [ -n "${fcnodes}" ] || [ -n "${ftnodes}" ]; then
     salt -C 'mas01*' state.apply maas.machines
     return 1
   fi
diff --git a/mcp/patches/0009-maas.py-Add-missing-status-ids.patch b/mcp/patches/0009-maas.py-Add-missing-status-ids.patch
new file mode 100644
index 000000000..648e62e51
--- /dev/null
+++ b/mcp/patches/0009-maas.py-Add-missing-status-ids.patch
@@ -0,0 +1,38 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 17 Dec 2017 06:30:27 +0100
+Subject: [PATCH] maas.py: Add missing status ids
+
+Status ID list synced from MaaS 2.3 [1].
+
+[1] https://github.com/maas/maas/blob/2.3/src/maasserver/\
+    migrations/builtin/maasserver/0106_testing_status.py#L25
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ _modules/maas.py | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/_modules/maas.py b/_modules/maas.py
+index e279b70..1df4698 100644
+--- a/_modules/maas.py
++++ b/_modules/maas.py
+@@ -52,7 +52,10 @@ def __virtual__():
+     (9, 'Deploying'), (6, 'Deployed'), (7, 'Retired'), (8, 'Broken'),
+     (11, 'Failed deployment'), (12, 'Releasing'),
+     (13, 'Releasing failed'), (14, 'Disk erasing'),
+-    (15, 'Failed disk erasing')])
++    (15, 'Failed disk erasing'), (16, 'Rescue mode'),
++    (17, 'Entering rescue mode'), (18, 'Failed to enter rescue mode'),
++    (19, 'Exiting rescue mode'), (20, 'Failed to exit rescue mode'),
++    (21, 'Testing'), (22, 'Failed testing')])
+
+
+ def _format_data(data):
diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
index 203013368..62b0f60d1 100644
--- a/mcp/patches/patches.list
+++ b/mcp/patches/patches.list
@@ -13,6 +13,7 @@
 /usr/share/salt-formulas/env: 0006-maas-module-Add-VLAN-DHCP-enable-support.patch
 /usr/share/salt-formulas/env: 0007-network.interface-Fix-ifup-OVS-port-with-route.patch
 /usr/share/salt-formulas/env: 0008-Handle-file_recv-option.patch
+/usr/share/salt-formulas/env: 0009-maas.py-Add-missing-status-ids.patch
 /usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch
 /usr/share/salt-formulas/reclass: 0011-service.horizon.server.cluster-Default-to-v2-API.patch
 /usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch
diff --git a/mcp/salt-formulas/maas/machines/delete.sls b/mcp/salt-formulas/maas/machines/delete.sls
index dbb625a02..2903f9226 100644
--- a/mcp/salt-formulas/maas/machines/delete.sls
+++ b/mcp/salt-formulas/maas/machines/delete.sls
@@ -10,6 +10,7 @@
 maas_login_admin:
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
 
 # TODO: implement delete_machine via _modules/maas.py
 delete_machine:
diff --git a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
index 17a7df8d8..3983729e7 100644
--- a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
+++ b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls
@@ -10,6 +10,7 @@
 maas_login_admin:
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
 
 # TODO: implement mark_broken_fixed_machine via _modules/maas.py
 mark_broken_fixed_machine:
diff --git a/mcp/salt-formulas/maas/machines/override_failed_testing.sls b/mcp/salt-formulas/maas/machines/override_failed_testing.sls
new file mode 100644
index 000000000..e7fe1d267
--- /dev/null
+++ b/mcp/salt-formulas/maas/machines/override_failed_testing.sls
@@ -0,0 +1,20 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+{%- from "maas/map.jinja" import region with context %}
+
+maas_login_admin:
+  cmd.run:
+  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
+  - unless: 'test -e /var/lib/maas/.maas_credentials'
+
+# TODO: implement override_failed_testing via _modules/maas.py
+mark_broken_fixed_machine:
+  cmd.run:
+  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}"
+  - require:
+    - cmd: maas_login_admin