# # Copyright (c) 2017 All rights reserved # This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # # http://www.apache.org/licenses/LICENSE-2.0 # # import utils_log as log import os import six import re import signal import subprocess from time import sleep from threading import Thread try: from Queue import Queue except ImportError: from queue import Queue # python 3.x LOG = log.LOG LOG_LEVEL = log.LOG_LEVEL def _subprocess_setup(): # Python installs a SIGPIPE handler by default. This is usually not what # non-Python subprocesses expect. signal.signal(signal.SIGPIPE, signal.SIG_DFL) # NOTE(flaper87): The following globals are used by `mask_password` _SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] # NOTE(ldbragst): Let's build a list of regex objects using the list of # _SANITIZE_KEYS we already have. This way, we only have to add the new key # to the list of _SANITIZE_KEYS and we can generate regular expressions # for XML and JSON automatically. _SANITIZE_PATTERNS_2 = [] _SANITIZE_PATTERNS_1 = [] def mask_password(message, secret="***"): """Replace password with 'secret' in message. :param message: The string which includes security information. :param secret: value with which to replace passwords. :returns: The unicode value of message with the password fields masked. For example: >>> mask_password("'adminPass' : 'aaaaa'") "'adminPass' : '***'" >>> mask_password("'admin_pass' : 'aaaaa'") "'admin_pass' : '***'" >>> mask_password('"password" : "aaaaa"') '"password" : "***"' >>> mask_password("'original_password' : 'aaaaa'") "'original_password' : '***'" >>> mask_password("u'original_password' : u'aaaaa'") "u'original_password' : u'***'" """ try: message = six.text_type(message) except UnicodeDecodeError: # NOTE(jecarey): Temporary fix to handle cases where message is a # byte string. A better solution will be provided in Kilo. pass # NOTE(ldbragst): Check to see if anything in message contains any key # specified in _SANITIZE_KEYS, if not then just return the message since # we don't have to mask any passwords. if not any(key in message for key in _SANITIZE_KEYS): return message substitute = r'\g<1>' + secret + r'\g<2>' for pattern in _SANITIZE_PATTERNS_2: message = re.sub(pattern, substitute, message) substitute = r'\g<1>' + secret for pattern in _SANITIZE_PATTERNS_1: message = re.sub(pattern, substitute, message) return message class ProcessExecutionError(Exception): def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, description=None): self.exit_code = exit_code self.stderr = stderr self.stdout = stdout self.cmd = cmd self.description = description if description is None: description = "Unexpected error while running command." if exit_code is None: exit_code = '-' message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % (description, cmd, exit_code, stdout, stderr)) super(ProcessExecutionError, self).__init__(message) def enqueue_output(out, queue): for line in iter(out.readline, b''): queue.put(line) queue.put("##Finished##") out.close() def execute(cmd, **kwargs): """Helper method to shell out and execute a command through subprocess. Allows optional retry. :param cmd: Passed to subprocess.Popen. :type cmd: list - will be converted if needed :param process_input: Send to opened process. :type proces_input: string :param check_exit_code: Single bool, int, or list of allowed exit codes. Defaults to [0]. Raise :class:`ProcessExecutionError` unless program exits with one of these code. :type check_exit_code: boolean, int, or [int] :param delay_on_retry: True | False. Defaults to True. If set to True, wait a short amount of time before retrying. :type delay_on_retry: boolean :param attempts: How many times to retry cmd. :type attempts: int :param run_as_root: True | False. Defaults to False. If set to True, or as_root the command is prefixed by the command specified in the root_helper kwarg. execute this command. Defaults to false. :param shell: whether or not there should be a shell used to :type shell: boolean :param loglevel: log level for execute commands. :type loglevel: int. (Should be logging.DEBUG or logging.INFO) :param non_blocking Execute in background. :type non_blockig: boolean :returns: (stdout, (stderr, returncode)) from process execution :raises: :class:`UnknownArgumentError` on receiving unknown arguments :raises: :class:`ProcessExecutionError` """ process_input = kwargs.pop('process_input', None) check_exit_code = kwargs.pop('check_exit_code', [0]) ignore_exit_code = False attempts = kwargs.pop('attempts', 1) run_as_root = kwargs.pop('run_as_root', False) or kwargs.pop('as_root', False) shell = kwargs.pop('shell', False) loglevel = kwargs.pop('loglevel', LOG_LEVEL) non_blocking = kwargs.pop('non_blocking', False) if not isinstance(cmd, list): cmd = cmd.split(' ') if run_as_root: cmd = ['sudo'] + cmd if shell: cmd = ' '.join(cmd) if isinstance(check_exit_code, bool): ignore_exit_code = not check_exit_code check_exit_code = [0] elif isinstance(check_exit_code, int): check_exit_code = [check_exit_code] if kwargs: raise Exception(('Got unknown keyword args ' 'to utils.execute: %r') % kwargs) while attempts > 0: attempts -= 1 try: LOG.log(loglevel, ('Running cmd (subprocess): %s'), cmd) _PIPE = subprocess.PIPE # pylint: disable=E1101 if os.name == 'nt': preexec_fn = None close_fds = False else: preexec_fn = _subprocess_setup close_fds = True obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, stderr=_PIPE, close_fds=close_fds, preexec_fn=preexec_fn, shell=shell) result = None if process_input is not None: result = obj.communicate(process_input) else: if non_blocking: queue = Queue() thread = Thread(target=enqueue_output, args=(obj.stdout, queue)) thread.deamon = True thread.start() # If you want to read this output later: # try: # from Queue import Queue, Empty # except ImportError: # from queue import Queue, Empty # python 3.x # try: line = q.get_nowait() # or q.get(timeout=.1) # except Empty: # print('no output yet') # else: # got line # ... do something with line return queue result = obj.communicate() obj.stdin.close() # pylint: disable=E1101 _returncode = obj.returncode # pylint: disable=E1101 LOG.log(loglevel, ('Result was %s') % _returncode) if not ignore_exit_code and _returncode not in check_exit_code: (stdout, stderr) = result sanitized_stdout = mask_password(stdout) sanitized_stderr = mask_password(stderr) raise ProcessExecutionError( exit_code=_returncode, stdout=sanitized_stdout, stderr=sanitized_stderr, cmd=(' '.join(cmd)) if isinstance(cmd, list) else cmd) (stdout, stderr) = result return stdout, (stderr, _returncode) except ProcessExecutionError: raise finally: sleep(0)