summaryrefslogtreecommitdiffstats
path: root/odl-pipeline/lib/utils/processutils.py
diff options
context:
space:
mode:
authorNiko Hermanns <nikolas.hermanns@ericsson.com>2016-11-30 11:59:44 +0100
committerNikolas Hermanns <nikolas.hermanns@ericsson.com>2016-12-13 11:26:00 +0100
commit416f9cf177801cecfe39f5249d73e3a4a85f0bbd (patch)
treebea48ec45edc10489aa76d348b0dbfbb1cf4d7ab /odl-pipeline/lib/utils/processutils.py
parentb9eb7024b014cba0d299b1cf3b01e179c7d0482e (diff)
adding odl-pipeline
Change-Id: I1c08883f0d68a61ce9e10c5596aec1a259eed71f Signed-off-by: Nikolas Hermanns <nikolas.hermanns@ericsson.com>
Diffstat (limited to 'odl-pipeline/lib/utils/processutils.py')
-rwxr-xr-xodl-pipeline/lib/utils/processutils.py233
1 files changed, 233 insertions, 0 deletions
diff --git a/odl-pipeline/lib/utils/processutils.py b/odl-pipeline/lib/utils/processutils.py
new file mode 100755
index 0000000..b5aecb3
--- /dev/null
+++ b/odl-pipeline/lib/utils/processutils.py
@@ -0,0 +1,233 @@
+#
+# Copyright (c) 2015 All rights reserved
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+#
+import utils_log as log
+import os
+import six
+import re
+import signal
+import subprocess
+from time import sleep
+from threading import Thread
+try:
+ from Queue import Queue
+except ImportError:
+ from queue import Queue # python 3.x
+
+LOG = log.LOG
+LOG_LEVEL = log.LOG_LEVEL
+
+
+def _subprocess_setup():
+ # Python installs a SIGPIPE handler by default. This is usually not what
+ # non-Python subprocesses expect.
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+# NOTE(flaper87): The following globals are used by `mask_password`
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS_2 = []
+_SANITIZE_PATTERNS_1 = []
+
+
+def mask_password(message, secret="***"):
+ """Replace password with 'secret' in message.
+
+ :param message: The string which includes security information.
+ :param secret: value with which to replace passwords.
+ :returns: The unicode value of message with the password fields masked.
+
+ For example:
+
+ >>> mask_password("'adminPass' : 'aaaaa'")
+ "'adminPass' : '***'"
+ >>> mask_password("'admin_pass' : 'aaaaa'")
+ "'admin_pass' : '***'"
+ >>> mask_password('"password" : "aaaaa"')
+ '"password" : "***"'
+ >>> mask_password("'original_password' : 'aaaaa'")
+ "'original_password' : '***'"
+ >>> mask_password("u'original_password' : u'aaaaa'")
+ "u'original_password' : u'***'"
+ """
+ try:
+ message = six.text_type(message)
+ except UnicodeDecodeError:
+ # NOTE(jecarey): Temporary fix to handle cases where message is a
+ # byte string. A better solution will be provided in Kilo.
+ pass
+
+ # NOTE(ldbragst): Check to see if anything in message contains any key
+ # specified in _SANITIZE_KEYS, if not then just return the message since
+ # we don't have to mask any passwords.
+ if not any(key in message for key in _SANITIZE_KEYS):
+ return message
+
+ substitute = r'\g<1>' + secret + r'\g<2>'
+ for pattern in _SANITIZE_PATTERNS_2:
+ message = re.sub(pattern, substitute, message)
+
+ substitute = r'\g<1>' + secret
+ for pattern in _SANITIZE_PATTERNS_1:
+ message = re.sub(pattern, substitute, message)
+
+ return message
+
+
+class ProcessExecutionError(Exception):
+ def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
+ description=None):
+ self.exit_code = exit_code
+ self.stderr = stderr
+ self.stdout = stdout
+ self.cmd = cmd
+ self.description = description
+
+ if description is None:
+ description = "Unexpected error while running command."
+ if exit_code is None:
+ exit_code = '-'
+ message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
+ % (description, cmd, exit_code, stdout, stderr))
+ super(ProcessExecutionError, self).__init__(message)
+
+
+def enqueue_output(out, queue):
+ for line in iter(out.readline, b''):
+ queue.put(line)
+ queue.put("##Finished##")
+ out.close()
+
+
+def execute(cmd, **kwargs):
+ """Helper method to shell out and execute a command through subprocess.
+
+ Allows optional retry.
+
+ :param cmd: Passed to subprocess.Popen.
+ :type cmd: list - will be converted if needed
+ :param process_input: Send to opened process.
+ :type proces_input: string
+ :param check_exit_code: Single bool, int, or list of allowed exit
+ codes. Defaults to [0]. Raise
+ :class:`ProcessExecutionError` unless
+ program exits with one of these code.
+ :type check_exit_code: boolean, int, or [int]
+ :param delay_on_retry: True | False. Defaults to True. If set to True,
+ wait a short amount of time before retrying.
+ :type delay_on_retry: boolean
+ :param attempts: How many times to retry cmd.
+ :type attempts: int
+ :param run_as_root: True | False. Defaults to False. If set to True,
+ or as_root the command is prefixed by the command specified
+ in the root_helper kwarg.
+ execute this command. Defaults to false.
+ :param shell: whether or not there should be a shell used to
+ :type shell: boolean
+ :param loglevel: log level for execute commands.
+ :type loglevel: int. (Should be logging.DEBUG or logging.INFO)
+ :param non_blocking Execute in background.
+ :type non_blockig: boolean
+ :returns: (stdout, (stderr, returncode)) from process
+ execution
+ :raises: :class:`UnknownArgumentError` on
+ receiving unknown arguments
+ :raises: :class:`ProcessExecutionError`
+ """
+ process_input = kwargs.pop('process_input', None)
+ check_exit_code = kwargs.pop('check_exit_code', [0])
+ ignore_exit_code = False
+ attempts = kwargs.pop('attempts', 1)
+ run_as_root = kwargs.pop('run_as_root', False) or kwargs.pop('as_root',
+ False)
+ shell = kwargs.pop('shell', False)
+ loglevel = kwargs.pop('loglevel', LOG_LEVEL)
+ non_blocking = kwargs.pop('non_blocking', False)
+
+ if not isinstance(cmd, list):
+ cmd = cmd.split(' ')
+
+ if run_as_root:
+ cmd = ['sudo'] + cmd
+ if shell:
+ cmd = ' '.join(cmd)
+ if isinstance(check_exit_code, bool):
+ ignore_exit_code = not check_exit_code
+ check_exit_code = [0]
+ elif isinstance(check_exit_code, int):
+ check_exit_code = [check_exit_code]
+
+ if kwargs:
+ raise Exception(('Got unknown keyword args '
+ 'to utils.execute: %r') % kwargs)
+
+ while attempts > 0:
+ attempts -= 1
+ try:
+ LOG.log(loglevel, ('Running cmd (subprocess): %s'), cmd)
+ _PIPE = subprocess.PIPE # pylint: disable=E1101
+
+ if os.name == 'nt':
+ preexec_fn = None
+ close_fds = False
+ else:
+ preexec_fn = _subprocess_setup
+ close_fds = True
+
+ obj = subprocess.Popen(cmd,
+ stdin=_PIPE,
+ stdout=_PIPE,
+ stderr=_PIPE,
+ close_fds=close_fds,
+ preexec_fn=preexec_fn,
+ shell=shell)
+ result = None
+ if process_input is not None:
+ result = obj.communicate(process_input)
+ else:
+ if non_blocking:
+ queue = Queue()
+ thread = Thread(target=enqueue_output, args=(obj.stdout,
+ queue))
+ thread.deamon = True
+ thread.start()
+ # If you want to read this output later:
+ # try:
+ # from Queue import Queue, Empty
+ # except ImportError:
+ # from queue import Queue, Empty # python 3.x
+ # try: line = q.get_nowait() # or q.get(timeout=.1)
+ # except Empty:
+ # print('no output yet')
+ # else: # got line
+ # ... do something with line
+ return queue
+ result = obj.communicate()
+ obj.stdin.close() # pylint: disable=E1101
+ _returncode = obj.returncode # pylint: disable=E1101
+ LOG.log(loglevel, ('Result was %s') % _returncode)
+ if not ignore_exit_code and _returncode not in check_exit_code:
+ (stdout, stderr) = result
+ sanitized_stdout = mask_password(stdout)
+ sanitized_stderr = mask_password(stderr)
+ raise ProcessExecutionError(
+ exit_code=_returncode,
+ stdout=sanitized_stdout,
+ stderr=sanitized_stderr,
+ cmd=(' '.join(cmd)) if isinstance(cmd, list) else cmd)
+ (stdout, stderr) = result
+ return (stdout, (stderr, _returncode))
+ except ProcessExecutionError:
+ raise
+ finally:
+ sleep(0)