Merge "adding odl-pipeline"

1 files changed, 233 insertions, 0 deletions

diff --git a/odl-pipeline/lib/utils/processutils.py b/odl-pipeline/lib/utils/processutils.py
new file mode 100755
index 0000000..b5aecb3
--- /dev/null
+++ b/odl-pipeline/lib/utils/processutils.py
@@ -0,0 +1,233 @@
+#
+# Copyright (c) 2015 All rights reserved
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+#
+import utils_log as log
+import os
+import six
+import re
+import signal
+import subprocess
+from time import sleep
+from threading import Thread
+try:
+    from Queue import Queue
+except ImportError:
+    from queue import Queue  # python 3.x
+
+LOG = log.LOG
+LOG_LEVEL = log.LOG_LEVEL
+
+
+def _subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+# NOTE(flaper87): The following globals are used by `mask_password`
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS_2 = []
+_SANITIZE_PATTERNS_1 = []
+
+
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords.
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    try:
+        message = six.text_type(message)
+    except UnicodeDecodeError:
+        # NOTE(jecarey): Temporary fix to handle cases where message is a
+        # byte string.   A better solution will be provided in Kilo.
+        pass
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    substitute = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS_2:
+        message = re.sub(pattern, substitute, message)
+
+    substitute = r'\g<1>' + secret
+    for pattern in _SANITIZE_PATTERNS_1:
+        message = re.sub(pattern, substitute, message)
+
+    return message
+
+
+class ProcessExecutionError(Exception):
+    def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
+                 description=None):
+        self.exit_code = exit_code
+        self.stderr = stderr
+        self.stdout = stdout
+        self.cmd = cmd
+        self.description = description
+
+        if description is None:
+            description = "Unexpected error while running command."
+        if exit_code is None:
+            exit_code = '-'
+        message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r"
+                   % (description, cmd, exit_code, stdout, stderr))
+        super(ProcessExecutionError, self).__init__(message)
+
+
+def enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+    queue.put("##Finished##")
+    out.close()
+
+
+def execute(cmd, **kwargs):
+    """Helper method to shell out and execute a command through subprocess.
+
+    Allows optional retry.
+
+    :param cmd:             Passed to subprocess.Popen.
+    :type cmd:              list - will be converted if needed
+    :param process_input:   Send to opened process.
+    :type proces_input:     string
+    :param check_exit_code: Single bool, int, or list of allowed exit
+                            codes.  Defaults to [0].  Raise
+                            :class:`ProcessExecutionError` unless
+                            program exits with one of these code.
+    :type check_exit_code:  boolean, int, or [int]
+    :param delay_on_retry:  True | False. Defaults to True. If set to True,
+                            wait a short amount of time before retrying.
+    :type delay_on_retry:   boolean
+    :param attempts:        How many times to retry cmd.
+    :type attempts:         int
+    :param run_as_root:     True | False. Defaults to False. If set to True,
+        or as_root          the command is prefixed by the command specified
+                            in the root_helper kwarg.
+                            execute this command. Defaults to false.
+    :param shell:           whether or not there should be a shell used to
+    :type shell:            boolean
+    :param loglevel:        log level for execute commands.
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
+    :param non_blocking     Execute in background.
+    :type non_blockig:      boolean
+    :returns:               (stdout, (stderr, returncode)) from process
+                            execution
+    :raises:                :class:`UnknownArgumentError` on
+                            receiving unknown arguments
+    :raises:                :class:`ProcessExecutionError`
+    """
+    process_input = kwargs.pop('process_input', None)
+    check_exit_code = kwargs.pop('check_exit_code', [0])
+    ignore_exit_code = False
+    attempts = kwargs.pop('attempts', 1)
+    run_as_root = kwargs.pop('run_as_root', False) or kwargs.pop('as_root',
+                                                                 False)
+    shell = kwargs.pop('shell', False)
+    loglevel = kwargs.pop('loglevel', LOG_LEVEL)
+    non_blocking = kwargs.pop('non_blocking', False)
+
+    if not isinstance(cmd, list):
+        cmd = cmd.split(' ')
+
+    if run_as_root:
+        cmd = ['sudo'] + cmd
+    if shell:
+        cmd = ' '.join(cmd)
+    if isinstance(check_exit_code, bool):
+        ignore_exit_code = not check_exit_code
+        check_exit_code = [0]
+    elif isinstance(check_exit_code, int):
+        check_exit_code = [check_exit_code]
+
+    if kwargs:
+        raise Exception(('Got unknown keyword args '
+                         'to utils.execute: %r') % kwargs)
+
+    while attempts > 0:
+        attempts -= 1
+        try:
+            LOG.log(loglevel, ('Running cmd (subprocess): %s'), cmd)
+            _PIPE = subprocess.PIPE  # pylint: disable=E1101
+
+            if os.name == 'nt':
+                preexec_fn = None
+                close_fds = False
+            else:
+                preexec_fn = _subprocess_setup
+                close_fds = True
+
+            obj = subprocess.Popen(cmd,
+                                   stdin=_PIPE,
+                                   stdout=_PIPE,
+                                   stderr=_PIPE,
+                                   close_fds=close_fds,
+                                   preexec_fn=preexec_fn,
+                                   shell=shell)
+            result = None
+            if process_input is not None:
+                result = obj.communicate(process_input)
+            else:
+                if non_blocking:
+                    queue = Queue()
+                    thread = Thread(target=enqueue_output, args=(obj.stdout,
+                                                                 queue))
+                    thread.deamon = True
+                    thread.start()
+                    # If you want to read this output later:
+                    # try:
+                    #     from Queue import Queue, Empty
+                    # except ImportError:
+                    #     from queue import Queue, Empty  # python 3.x
+                    # try:  line = q.get_nowait() # or q.get(timeout=.1)
+                    # except Empty:
+                    #     print('no output yet')
+                    # else: # got line
+                    # ... do something with line
+                    return queue
+                result = obj.communicate()
+            obj.stdin.close()  # pylint: disable=E1101
+            _returncode = obj.returncode  # pylint: disable=E1101
+            LOG.log(loglevel, ('Result was %s') % _returncode)
+            if not ignore_exit_code and _returncode not in check_exit_code:
+                (stdout, stderr) = result
+                sanitized_stdout = mask_password(stdout)
+                sanitized_stderr = mask_password(stderr)
+                raise ProcessExecutionError(
+                    exit_code=_returncode,
+                    stdout=sanitized_stdout,
+                    stderr=sanitized_stderr,
+                    cmd=(' '.join(cmd)) if isinstance(cmd, list) else cmd)
+            (stdout, stderr) = result
+            return (stdout, (stderr, _returncode))
+        except ProcessExecutionError:
+            raise
+        finally:
+            sleep(0)