diff options
| -rw-r--r-- | deploy/common.py | 24 | ||||
| -rw-r--r-- | deploy/dha_adapters/ipmi_adapter.py | 101 |
2 files changed, 59 insertions, 66 deletions
diff --git a/deploy/common.py b/deploy/common.py index 41b4e274e..3cd3e0e6e 100644 --- a/deploy/common.py +++ b/deploy/common.py @@ -16,6 +16,7 @@ import argparse import shutil import stat import errno +import time N = {'id': 0, 'status': 1, 'name': 2, 'cluster': 3, 'ip': 4, 'mac': 5, 'roles': 6, 'pending_roles': 7, 'online': 8, 'group_id': 9} @@ -37,13 +38,22 @@ out_handler.setFormatter(formatter) LOG.addHandler(out_handler) os.chmod(LOGFILE, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) -def exec_cmd(cmd, check=True): - process = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True) - (response, stderr) = process.communicate() - return_code = process.returncode +def exec_cmd(cmd, check=True, attempts=1, delay=5, verbose=False): + # a negative value means forever + while attempts != 0: + attempts = attempts - 1 + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + (response, stderr) = process.communicate() + return_code = process.returncode + if return_code == 0 or attempts == 0: + break + time.sleep(delay) + if verbose: + log('%d attempts left: %s' % (attempts, cmd)) + response = response.strip() if check: if return_code > 0: diff --git a/deploy/dha_adapters/ipmi_adapter.py b/deploy/dha_adapters/ipmi_adapter.py index 4bd6bd378..6ce4012f4 100644 --- a/deploy/dha_adapters/ipmi_adapter.py +++ b/deploy/dha_adapters/ipmi_adapter.py @@ -1,5 +1,6 @@ ############################################################################### # Copyright (c) 2015 Ericsson AB and others. +# (c) 2016 Enea Software AB # szilard.cserey@ericsson.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -20,8 +21,10 @@ from common import ( class IpmiAdapter(HardwareAdapter): - def __init__(self, yaml_path): + def __init__(self, yaml_path, attempts=20, delay=3): super(IpmiAdapter, self).__init__(yaml_path) + self.attempts = attempts + self.delay = delay def get_access_info(self, node_id): ip = self.get_node_property(node_id, 'ipmiIp') @@ -43,69 +46,46 @@ class IpmiAdapter(HardwareAdapter): mac_list.append(self.get_node_property(node_id, 'pxeMac').lower()) return mac_list + def node_get_state(self, node_id): + state = exec_cmd('%s chassis power status' % self.ipmi_cmd(node_id), + attempts=self.attempts, delay=self.delay, + verbose=True) + return state + + def _node_power_cmd(self, node_id, cmd): + expected = 'Chassis Power is %s' % cmd + if self.node_get_state(node_id) == expected: + return + + pow_cmd = '%s chassis power %s' % (self.ipmi_cmd(node_id), cmd) + exec_cmd(pow_cmd, attempts=self.attempts, delay=self.delay, + verbose=True) + + attempts = self.attempts + while attempts: + state = self.node_get_state(node_id) + attempts -= 1 + if state == expected: + return + elif attempts != 0: + # reinforce our will, but allow the command to fail, + # we know our message got across once already... + exec_cmd(pow_cmd, check=False) + + err('Could not set chassis %s for node %s' % (cmd, node_id)) + def node_power_on(self, node_id): - WAIT_LOOP = 200 - SLEEP_TIME = 3 log('Power ON Node %s' % node_id) - cmd_prefix = self.ipmi_cmd(node_id) - state = exec_cmd('%s chassis power status' % cmd_prefix) - if state == 'Chassis Power is off': - exec_cmd('%s chassis power on' % cmd_prefix) - done = False - for i in range(WAIT_LOOP): - state, _ = exec_cmd('%s chassis power status' % cmd_prefix, - False) - if state == 'Chassis Power is on': - done = True - break - else: - time.sleep(SLEEP_TIME) - if not done: - err('Could Not Power ON Node %s' % node_id) + self._node_power_cmd(node_id, 'on') def node_power_off(self, node_id): - WAIT_LOOP = 200 - SLEEP_TIME = 3 log('Power OFF Node %s' % node_id) - cmd_prefix = self.ipmi_cmd(node_id) - state = exec_cmd('%s chassis power status' % cmd_prefix) - if state == 'Chassis Power is on': - done = False - exec_cmd('%s chassis power off' % cmd_prefix) - for i in range(WAIT_LOOP): - state, _ = exec_cmd('%s chassis power status' % cmd_prefix, - False) - if state == 'Chassis Power is off': - done = True - break - else: - time.sleep(SLEEP_TIME) - if not done: - err('Could Not Power OFF Node %s' % node_id) + self._node_power_cmd(node_id, 'off') def node_reset(self, node_id): - WAIT_LOOP = 600 log('RESET Node %s' % node_id) - cmd_prefix = self.ipmi_cmd(node_id) - state = exec_cmd('%s chassis power status' % cmd_prefix) - if state == 'Chassis Power is on': - was_shut_off = False - done = False - exec_cmd('%s chassis power reset' % cmd_prefix) - for i in range(WAIT_LOOP): - state, _ = exec_cmd('%s chassis power status' % cmd_prefix, - False) - if state == 'Chassis Power is off': - was_shut_off = True - elif state == 'Chassis Power is on' and was_shut_off: - done = True - break - time.sleep(1) - if not done: - err('Could Not RESET Node %s' % node_id) - else: - err('Cannot RESET Node %s because it\'s not Active, state: %s' - % (node_id, state)) + cmd = '%s chassis power reset' % self.ipmi_cmd(node_id) + exec_cmd(cmd, attempts=self.attempts, delay=self.delay, verbose=True) def node_set_boot_order(self, node_id, boot_order_list): log('Set boot order %s on Node %s' % (boot_order_list, node_id)) @@ -114,9 +94,12 @@ class IpmiAdapter(HardwareAdapter): for dev in boot_order_list: if dev == 'pxe': exec_cmd('%s chassis bootdev pxe options=persistent' - % cmd_prefix) + % cmd_prefix, attempts=self.attempts, delay=self.delay, + verbose=True) elif dev == 'iso': - exec_cmd('%s chassis bootdev cdrom' % cmd_prefix) + exec_cmd('%s chassis bootdev cdrom' % cmd_prefix, + attempts=self.attempts, delay=self.delay, verbose=True) elif dev == 'disk': exec_cmd('%s chassis bootdev disk options=persistent' - % cmd_prefix) + % cmd_prefix, attempts=self.attempts, delay=self.delay, + verbose=True) |