From 1c2b1de8cda572e5462113220206f0d934ac5f96 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Wed, 29 Jan 2020 17:31:06 +0100 Subject: maas: Avoid race condition in node fixups When more than one node enters a failure state during a deploy attempt, we recover the first one and issue another deploy request; avoid raising an exception for the second node (which is not in 'Ready' state either), allowing the retries to continue. Change-Id: I4a3e037e78b5c48aebf6e700115c0bbf848c7cd5 Signed-off-by: Alexandru Avadanii --- ...end-wait_for-maas.py-wait_for_-attempts-arg.patch | 20 +++++++++++++++----- .../0005-Implement-tags-support.patch | 4 ++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch b/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch index 1be210f3c..efd7cc2fd 100644 --- a/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch +++ b/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch @@ -60,13 +60,13 @@ Signed-off-by: ting wu Signed-off-by: Alexandru Avadanii --- README.rst | 9 +++- - _modules/maas.py | 49 ++++++++++++++++++-- + _modules/maas.py | 52 +++++++++++++++++--- maas/machines/wait_for_deployed.sls | 2 + maas/machines/wait_for_ready.sls | 3 ++ maas/machines/wait_for_ready_or_deployed.sls | 15 ++++++ maas/map.jinja | 4 ++ tests/pillar/maas_region.sls | 4 ++ - 7 files changed, 79 insertions(+), 7 deletions(-) + 7 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 maas/machines/wait_for_ready_or_deployed.sls diff --git a/README.rst b/README.rst @@ -102,9 +102,18 @@ index 20da43e..78d8aef 100644 ignore_machines: - kvm01 # in case it's broken or whatever diff --git a/_modules/maas.py b/_modules/maas.py -index c02f104..28e46c5 100644 +index c02f104..bb70576 100644 --- a/_modules/maas.py +++ b/_modules/maas.py +@@ -670,7 +670,7 @@ class DeployMachines(MaasObject): + if machine['status'] == self.DEPLOYED: + return + if machine['status'] != self.READY: +- raise Exception('Not in ready state') ++ return + data = { + 'system_id': machine['system_id'], + } @@ -921,6 +921,7 @@ class MachinesStatus(MaasObject): req_status: string; Polling status machines: list; machine names @@ -122,7 +131,7 @@ index c02f104..28e46c5 100644 if not to_discover: try: to_discover = __salt__['config.get']('maas')['region'][ -@@ -943,11 +946,45 @@ class MachinesStatus(MaasObject): +@@ -943,11 +946,46 @@ class MachinesStatus(MaasObject): while len(total) <= len(to_discover): for m in to_discover: for discovered in MachinesStatus.execute()['machines']: @@ -155,6 +164,7 @@ index c02f104..28e46c5 100644 + LOG.info('Machine {0} mark broken'.format(sid)) + cls._maas.post(u'api/2.0/machines/{0}/' + .format(sid), 'mark_broken', **data) ++ time.sleep(poll_time) + LOG.info('Machine {0} mark fixed'.format(sid)) + cls._maas.post(u'api/2.0/machines/{0}/' + .format(sid), 'mark_fixed', **data) @@ -172,7 +182,7 @@ index c02f104..28e46c5 100644 if len(total) <= 0: LOG.debug( "Machines:{} are:{}".format(to_discover, req_status)) -@@ -959,7 +996,9 @@ class MachinesStatus(MaasObject): +@@ -959,7 +997,9 @@ class MachinesStatus(MaasObject): "Waiting status:{} " "for machines:{}" "\nsleep for:{}s " diff --git a/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch b/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch index e59c2f2e8..09d21d022 100644 --- a/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch +++ b/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch @@ -34,7 +34,7 @@ index 78d8aef..8c35458 100644 # domain: mydomain.local # This function broken http_proxy: http://192.168.0.10:3142 diff --git a/_modules/maas.py b/_modules/maas.py -index 28e46c5..f56ed34 100644 +index bb70576..0cda8dd 100644 --- a/_modules/maas.py +++ b/_modules/maas.py @@ -876,6 +876,28 @@ class Domain(MaasObject): @@ -66,7 +66,7 @@ index 28e46c5..f56ed34 100644 class MachinesStatus(MaasObject): @classmethod def execute(cls, objects_name=None): -@@ -1064,5 +1086,9 @@ def process_sshprefs(): +@@ -1065,5 +1087,9 @@ def process_sshprefs(): return SSHPrefs().process() -- cgit 1.2.3-korg