diff options
Diffstat (limited to 'odl-pipeline/lib/utils/processutils.py')
| -rwxr-xr-x | odl-pipeline/lib/utils/processutils.py | 233 |
1 files changed, 233 insertions, 0 deletions
diff --git a/odl-pipeline/lib/utils/processutils.py b/odl-pipeline/lib/utils/processutils.py new file mode 100755 index 0000000..b5aecb3 --- /dev/null +++ b/odl-pipeline/lib/utils/processutils.py @@ -0,0 +1,233 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import utils_log as log +import os +import six +import re +import signal +import subprocess +from time import sleep +from threading import Thread +try: + from Queue import Queue +except ImportError: + from queue import Queue # python 3.x + +LOG = log.LOG +LOG_LEVEL = log.LOG_LEVEL + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +# NOTE(flaper87): The following globals are used by `mask_password` +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS_2 = [] +_SANITIZE_PATTERNS_1 = [] + + +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords. + :returns: The unicode value of message with the password fields masked. + + For example: + + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + try: + message = six.text_type(message) + except UnicodeDecodeError: + # NOTE(jecarey): Temporary fix to handle cases where message is a + # byte string. A better solution will be provided in Kilo. + pass + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + substitute = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS_2: + message = re.sub(pattern, substitute, message) + + substitute = r'\g<1>' + secret + for pattern in _SANITIZE_PATTERNS_1: + message = re.sub(pattern, substitute, message) + + return message + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" + % (description, cmd, exit_code, stdout, stderr)) + super(ProcessExecutionError, self).__init__(message) + + +def enqueue_output(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + queue.put("##Finished##") + out.close() + + +def execute(cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: list - will be converted if needed + :param process_input: Send to opened process. + :type proces_input: string + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + or as_root the command is prefixed by the command specified + in the root_helper kwarg. + execute this command. Defaults to false. + :param shell: whether or not there should be a shell used to + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be logging.DEBUG or logging.INFO) + :param non_blocking Execute in background. + :type non_blockig: boolean + :returns: (stdout, (stderr, returncode)) from process + execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) or kwargs.pop('as_root', + False) + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', LOG_LEVEL) + non_blocking = kwargs.pop('non_blocking', False) + + if not isinstance(cmd, list): + cmd = cmd.split(' ') + + if run_as_root: + cmd = ['sudo'] + cmd + if shell: + cmd = ' '.join(cmd) + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise Exception(('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, ('Running cmd (subprocess): %s'), cmd) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + if process_input is not None: + result = obj.communicate(process_input) + else: + if non_blocking: + queue = Queue() + thread = Thread(target=enqueue_output, args=(obj.stdout, + queue)) + thread.deamon = True + thread.start() + # If you want to read this output later: + # try: + # from Queue import Queue, Empty + # except ImportError: + # from queue import Queue, Empty # python 3.x + # try: line = q.get_nowait() # or q.get(timeout=.1) + # except Empty: + # print('no output yet') + # else: # got line + # ... do something with line + return queue + result = obj.communicate() + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, ('Result was %s') % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + sanitized_stdout = mask_password(stdout) + sanitized_stderr = mask_password(stderr) + raise ProcessExecutionError( + exit_code=_returncode, + stdout=sanitized_stdout, + stderr=sanitized_stderr, + cmd=(' '.join(cmd)) if isinstance(cmd, list) else cmd) + (stdout, stderr) = result + return (stdout, (stderr, _returncode)) + except ProcessExecutionError: + raise + finally: + sleep(0) |