diff options
Diffstat (limited to 'odl-pipeline/lib/utils')
| -rwxr-xr-x | odl-pipeline/lib/utils/__init__.py | 0 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/node.py | 87 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/node_manager.py | 43 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/processutils.py | 233 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/service.py | 67 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/shutil.py | 66 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/ssh_client.py | 88 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/ssh_util.py | 36 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/utils_log.py | 67 | ||||
| -rwxr-xr-x | odl-pipeline/lib/utils/utils_yaml.py | 20 |
10 files changed, 707 insertions, 0 deletions
diff --git a/odl-pipeline/lib/utils/__init__.py b/odl-pipeline/lib/utils/__init__.py new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/odl-pipeline/lib/utils/__init__.py diff --git a/odl-pipeline/lib/utils/node.py b/odl-pipeline/lib/utils/node.py new file mode 100755 index 0000000..c3c2005 --- /dev/null +++ b/odl-pipeline/lib/utils/node.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +from ssh_client import SSHClient +from ssh_util import SshUtil +from utils_log import log_enter_exit, for_all_methods + + +@for_all_methods(log_enter_exit) +class Node(object): + + def __init__(self, name, address=None, port=None, + user=None, password=None, jump=None, dict=None): + self.name = name + self.address = address + self.jump = jump + self.user = user + self.port = port + self.password = password + if dict: + self.read_from_dic(dict) + self.sshc = SSHClient(self) + self.has_access = False + self.config = dict + + def read_from_dic(self, dic): + allowed_keys = ['address', 'user', 'jump', 'password', 'port'] + for (key, value) in dic.iteritems(): + if key in allowed_keys: + setattr(self, key, value) + + def ping(self, ip): + self.execute(['ping', '-c', '1', ip]) + + def execute(self, cmd, **kwargs): + return self.sshc.execute(cmd, **kwargs) + + def chown(self, user, path): + self.execute('chown -R %(user)s:%(user)s %(path)s' % {'user': user, + 'path': path}, + as_root=True) + + def is_dir(self, path): + rv, _ = self.execute('test -d %s && echo yes' % path, + check_exit_code=[0, 1]) + if rv == 'yes\n': + return True + else: + return False + + def is_file(self, path): + rv, _ = self.execute('test -f %s && echo yes' % path, + check_exit_code=[0, 1]) + if rv == 'yes\n': + return True + else: + return False + + def reboot(self): + self.execute('reboot', as_root=True, check_exit_code=[255]) + + def create_path_if_not_exsist(self, path, **kwargs): + return self.sshc.execute('mkdir -p %s' % path, **kwargs) + + def copy(self, direction, local_path, remote_path, **kwargs): + return self.sshc.copy(direction, local_path, remote_path, **kwargs) + + def to_ssh_config(self): + config = ["Host %s" % self.name, + " Hostname %s" % + (self.address if self.address else self.name)] + if self.jump: + config.append(" ProxyCommand ssh -F %(config_path)s " + "-W %%h:%%p %(name)s" + % {'config_path': SshUtil.get_config_file_path(), + 'name': self.jump.name}) + if self.user: + config.append(" user %s" % self.user) + if self.port: + config.append(" port %s" % self.port) + return '\n'.join(config) diff --git a/odl-pipeline/lib/utils/node_manager.py b/odl-pipeline/lib/utils/node_manager.py new file mode 100755 index 0000000..d11065f --- /dev/null +++ b/odl-pipeline/lib/utils/node_manager.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +from ssh_util import SshUtil + + +class NodeManager(object): + + env_nodes = [] + env_node_dict = {} + primary_controller = None + + def __init__(self, config=None): + if config: + for (node_name, node_config) in config.iteritems(): + self.add_node(node_name, node_config) + + def add_node(self, node_name, node_config): + from node import Node + if not node_config.get('address'): + node_config['address'] = self.get_address_of_node(node_name) + node = Node(node_name, dict=node_config) + self.env_nodes.append(node) + self.env_node_dict[node_name] = node + return node + + def get_nodes(self): + return self.env_nodes + + def get_node(self, name): + return self.env_node_dict[name] + + @classmethod + def gen_ssh_config(cls, node): + if node not in cls.env_nodes: + cls.env_nodes.append(node) + SshUtil.gen_ssh_config(cls.env_nodes) diff --git a/odl-pipeline/lib/utils/processutils.py b/odl-pipeline/lib/utils/processutils.py new file mode 100755 index 0000000..b5aecb3 --- /dev/null +++ b/odl-pipeline/lib/utils/processutils.py @@ -0,0 +1,233 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import utils_log as log +import os +import six +import re +import signal +import subprocess +from time import sleep +from threading import Thread +try: + from Queue import Queue +except ImportError: + from queue import Queue # python 3.x + +LOG = log.LOG +LOG_LEVEL = log.LOG_LEVEL + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +# NOTE(flaper87): The following globals are used by `mask_password` +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS_2 = [] +_SANITIZE_PATTERNS_1 = [] + + +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords. + :returns: The unicode value of message with the password fields masked. + + For example: + + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + try: + message = six.text_type(message) + except UnicodeDecodeError: + # NOTE(jecarey): Temporary fix to handle cases where message is a + # byte string. A better solution will be provided in Kilo. + pass + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + substitute = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS_2: + message = re.sub(pattern, substitute, message) + + substitute = r'\g<1>' + secret + for pattern in _SANITIZE_PATTERNS_1: + message = re.sub(pattern, substitute, message) + + return message + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = ("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" + % (description, cmd, exit_code, stdout, stderr)) + super(ProcessExecutionError, self).__init__(message) + + +def enqueue_output(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + queue.put("##Finished##") + out.close() + + +def execute(cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: list - will be converted if needed + :param process_input: Send to opened process. + :type proces_input: string + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + or as_root the command is prefixed by the command specified + in the root_helper kwarg. + execute this command. Defaults to false. + :param shell: whether or not there should be a shell used to + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be logging.DEBUG or logging.INFO) + :param non_blocking Execute in background. + :type non_blockig: boolean + :returns: (stdout, (stderr, returncode)) from process + execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) or kwargs.pop('as_root', + False) + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', LOG_LEVEL) + non_blocking = kwargs.pop('non_blocking', False) + + if not isinstance(cmd, list): + cmd = cmd.split(' ') + + if run_as_root: + cmd = ['sudo'] + cmd + if shell: + cmd = ' '.join(cmd) + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise Exception(('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, ('Running cmd (subprocess): %s'), cmd) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + if process_input is not None: + result = obj.communicate(process_input) + else: + if non_blocking: + queue = Queue() + thread = Thread(target=enqueue_output, args=(obj.stdout, + queue)) + thread.deamon = True + thread.start() + # If you want to read this output later: + # try: + # from Queue import Queue, Empty + # except ImportError: + # from queue import Queue, Empty # python 3.x + # try: line = q.get_nowait() # or q.get(timeout=.1) + # except Empty: + # print('no output yet') + # else: # got line + # ... do something with line + return queue + result = obj.communicate() + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, ('Result was %s') % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + sanitized_stdout = mask_password(stdout) + sanitized_stderr = mask_password(stderr) + raise ProcessExecutionError( + exit_code=_returncode, + stdout=sanitized_stdout, + stderr=sanitized_stderr, + cmd=(' '.join(cmd)) if isinstance(cmd, list) else cmd) + (stdout, stderr) = result + return (stdout, (stderr, _returncode)) + except ProcessExecutionError: + raise + finally: + sleep(0) diff --git a/odl-pipeline/lib/utils/service.py b/odl-pipeline/lib/utils/service.py new file mode 100755 index 0000000..39cdce5 --- /dev/null +++ b/odl-pipeline/lib/utils/service.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import sys +import yaml +import argparse +import traceback +from utils_log import LOG, LOG_PATH +from abc import abstractmethod + + +class Service(object): + + def start(self): + try: + self._run() + except Exception as ex: + LOG.error(ex.message) + LOG.error(traceback.format_exc()) + LOG.error("For more logs check: %(log_path)s" + % {'log_path': LOG_PATH}) + sys.exit(1) + + def _run(self): + parser = self._create_cli_parser() + sys_args = parser.parse_args() + config = self.read_config(sys_args) + self.run(sys_args, config) + + @abstractmethod + def run(self, sys_args, config): + # Do something + return + + @abstractmethod + def create_cli_parser(self, parser): + # Read in own sys args + return parser + + def _create_cli_parser(self): + parser = argparse.ArgumentParser(description='OVS Debugger') + # parser.add_argument('-c', '--config', help="Path to config.yaml", + # required=False) + # parser.add_argument('--boolean', help="", + # required=False, action='store_true') + return self.create_cli_parser(parser) + + def read_config(self, sys_args): + if not hasattr(sys_args, 'config'): + return None + if not sys_args.config: + config_path = './etc/config.yaml' + else: + config_path = sys_args.config + try: + with open(config_path) as f: + return yaml.load(f) + except yaml.scanner.ScannerError as ex: + LOG.error("Yaml file corrupt. Try putting spaces after the " + "colons.") + raise ex diff --git a/odl-pipeline/lib/utils/shutil.py b/odl-pipeline/lib/utils/shutil.py new file mode 100755 index 0000000..40e2aba --- /dev/null +++ b/odl-pipeline/lib/utils/shutil.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import os +import glob +import processutils as putils + + +class shutil(): + ''' + classdocs + ''' + @staticmethod + def mkdir_if_not_exsist(path): + if not path: + raise Exception('Path should not be empty.') + putils.execute(["mkdir", "-p", path]) + + @staticmethod + def copy(direction, src, dst, **kwargs): + if direction == 'from': + dst_tmp = dst + dst = src + src = dst_tmp + if src[-1:] == '*': + files = glob.glob(src) + for file in files: + shutil._copy(file, dst, **kwargs) + else: + shutil._copy(src, dst, **kwargs) + + @staticmethod + def _copy(src, dst, **kwargs): + if os.path.isfile(src): + if dst[-1:] == '/': + shutil.mkdir_if_not_exsist(dst) + putils.execute(['cp', src, dst], **kwargs) + else: + putils.execute(['cp', '-R', src, dst], **kwargs) + + @staticmethod + def rm(path, **kwargs): + putils.execute(['rm', '-rf', path], **kwargs) + + @staticmethod + def mv(src, dst): + putils.execute(["mv", src, dst]) + + @staticmethod + def get_all_files_in_path(path): + if os.path.exists(path): + return putils.execute(['l', path]) + + @staticmethod + def replace_string_in_file(file, str, replace): + with open(file, 'r') as f: + string = f.read() + string = string.replace(str, replace) + with open(file, 'w+') as f: + f.write(string) diff --git a/odl-pipeline/lib/utils/ssh_client.py b/odl-pipeline/lib/utils/ssh_client.py new file mode 100755 index 0000000..464a74e --- /dev/null +++ b/odl-pipeline/lib/utils/ssh_client.py @@ -0,0 +1,88 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +from processutils import execute +from ssh_util import SshUtil +from node_manager import NodeManager +import os +from utils_log import LOG +import glob + + +class SSHClient(object): + + def __init__(self, node): + self.node = node + + def execute(self, cmd, **kwargs): + if 'log_true' in kwargs: + if kwargs['log_true']: + LOG.info('Node: %s Executing: %s' % (self.node.name, cmd)) + kwargs.pop('log_true') + NodeManager.gen_ssh_config(self.node) + if not isinstance(cmd, str): + cmd = ' '.join(cmd) + cmd_addition = ['ssh', '-i', SshUtil.get_id_rsa(), '-F', + SshUtil.get_config_file_path(), + self.node.name] + if self.node.password: + cmd_addition = ['sshpass', '-p', self.node.password] + cmd_addition + if 'as_root' in kwargs: + kwargs.pop('as_root') + cmd = 'sudo ' + cmd + cmd_addition.append(cmd) + return execute(cmd_addition, **kwargs) + + def copy(self, direction, local_path, remote_path, **kwargs): + all_files = None + if direction is 'to': + msg = ('Copying file %s to %s:%s' % (local_path, self.node.name, + remote_path)) + if self.node.is_dir(remote_path): + pass + elif remote_path[-1:] == '/': + self.node.create_path_if_not_exsist(remote_path) + else: + # Remove the file + self.execute('rm -f %s' % remote_path, as_root=True) + self.node.create_path_if_not_exsist( + os.path.dirname(remote_path)) + if '*' in local_path: + all_files = glob.glob(local_path) + else: + if local_path[-1:] == '/': + execute('mkdir -p %s' % local_path) + msg = ('Copying file from %s:%s to %s' % (self.node.name, + remote_path, + local_path)) + LOG.info(msg) + if all_files: + for one_file in all_files: + return self._copy(direction, one_file, remote_path, **kwargs) + else: + return self._copy(direction, local_path, remote_path, **kwargs) + + def _copy(self, direction, local_path, remote_path, **kwargs): + # TODO create dir is not existing + NodeManager.gen_ssh_config(self.node) + cmd = ['scp', '-i', SshUtil.get_id_rsa(), '-F', + SshUtil.get_config_file_path()] + if direction == 'to': + if os.path.isdir(local_path): + cmd.append('-r') + cmd = cmd + [local_path, + ('%s:%s') % (self.node.name, remote_path)] + if direction == 'from': + if self.node.is_dir(remote_path): + cmd.append('-r') + cmd = cmd + [('%s:%s') % (self.node.name, remote_path), + local_path] + if self.node.password: + cmd = ['sshpass', '-p', self.node.password] + cmd + return execute(cmd, **kwargs) diff --git a/odl-pipeline/lib/utils/ssh_util.py b/odl-pipeline/lib/utils/ssh_util.py new file mode 100755 index 0000000..e70aed3 --- /dev/null +++ b/odl-pipeline/lib/utils/ssh_util.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import os +home = os.getenv("HOME") +SSH_CONFIG = {'TMP_SSH_CONFIG': "./tmp/ssh_config", + 'ID_RSA_PATH': "%s/.ssh/id_rsa" % home} + + +class SshUtil(object): + + @staticmethod + def gen_ssh_config(node_list): + config = ["UserKnownHostsFile=/dev/null", + "StrictHostKeyChecking=no", + "ForwardAgent yes", + "GSSAPIAuthentication=no", + "LogLevel ERROR"] + for node in node_list: + config.append(node.to_ssh_config()) + with open(SSH_CONFIG['TMP_SSH_CONFIG'], 'w') as f: + f.write('\n'.join(config)) + + @staticmethod + def get_config_file_path(): + return SSH_CONFIG['TMP_SSH_CONFIG'] + + @staticmethod + def get_id_rsa(): + return (SSH_CONFIG['ID_RSA_PATH']) diff --git a/odl-pipeline/lib/utils/utils_log.py b/odl-pipeline/lib/utils/utils_log.py new file mode 100755 index 0000000..e49434c --- /dev/null +++ b/odl-pipeline/lib/utils/utils_log.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import logging +import datetime +import os +import sys + +LOG = logging.getLogger(__name__) +LOG_LEVEL = logging.DEBUG +LOG_PATH = "./tmp/%s.log" % os.path.basename(sys.argv[0]) +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', + filename=LOG_PATH, level=LOG_LEVEL) +# datefmt='%Y-%m-%dT:%H:%M:%s', level=LOG_LEVEL) +console = logging.StreamHandler() +console.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') +console.setFormatter(formatter) +LOG.addHandler(console) + + +def log_enter_exit(func): + + def inner(self, *args, **kwargs): + LOG.debug(("Entering %(cls)s.%(method)s " + "args: %(args)s, kwargs: %(kwargs)s") % + {'cls': self.__class__.__name__, + 'method': func.__name__, + 'args': args, + 'kwargs': kwargs}) + start = datetime.datetime.now() + ret = func(self, *args, **kwargs) + end = datetime.datetime.now() + LOG.debug(("Exiting %(cls)s.%(method)s. " + "Spent %(duration)s sec. " + "Return %(return)s") % + {'cls': self.__class__.__name__, + 'duration': end - start, + 'method': func.__name__, + 'return': ret}) + return ret + return inner + + +def for_all_methods(decorator): + # @for_all_methods(log_enter_exit) + # class ... + + def decorate(cls): + for attr in cls.__dict__: + if callable(getattr(cls, attr)): + setattr(cls, attr, decorator(getattr(cls, attr))) + return cls + return decorate + + +def dict_to_nice_string(dict): + return_string = [] + for key, value in dict.iteritems(): + return_string.append('%s: %s' % (key, value)) + return ', '.join(return_string) diff --git a/odl-pipeline/lib/utils/utils_yaml.py b/odl-pipeline/lib/utils/utils_yaml.py new file mode 100755 index 0000000..f9513b8 --- /dev/null +++ b/odl-pipeline/lib/utils/utils_yaml.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# +import yaml + + +def write_dict_to_yaml(config, path): + with open(path, 'w+') as f: + yaml.dump(config, f, default_flow_style=False) + + +def read_dict_from_yaml(path): + with open(path, 'r') as f: + return yaml.load(f) |