diff options
Diffstat (limited to 'compass-deck/utils')
| -rw-r--r-- | compass-deck/utils/__init__.py | 13 | ||||
| -rw-r--r-- | compass-deck/utils/celeryconfig_wrapper.py | 44 | ||||
| -rw-r--r-- | compass-deck/utils/daemonize.py | 76 | ||||
| -rw-r--r-- | compass-deck/utils/flags.py | 91 | ||||
| -rw-r--r-- | compass-deck/utils/logsetting.py | 108 | ||||
| -rw-r--r-- | compass-deck/utils/setting_wrapper.py | 175 | ||||
| -rw-r--r-- | compass-deck/utils/util.py | 395 |
7 files changed, 902 insertions, 0 deletions
diff --git a/compass-deck/utils/__init__.py b/compass-deck/utils/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-deck/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-deck/utils/celeryconfig_wrapper.py b/compass-deck/utils/celeryconfig_wrapper.py new file mode 100644 index 0000000..b6644ba --- /dev/null +++ b/compass-deck/utils/celeryconfig_wrapper.py @@ -0,0 +1,44 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""celeryconfig wrapper. + + .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com> +""" +import logging +import os.path +import urllib + +from compass.utils import setting_wrapper as setting + + +# CELERY_RESULT_BACKEND = 'amqp://' + +# BROKER_URL = 'amqp://guest:guest@localhost:5672//' + + +CELERY_IMPORTS = ('compass.tasks.tasks',) + + +if setting.CELERYCONFIG_FILE: + CELERY_CONFIG = os.path.join( + str(setting.CELERYCONFIG_DIR), + str(setting.CELERYCONFIG_FILE)) + + try: + logging.info('load celery config from %s', CELERY_CONFIG) + execfile(CELERY_CONFIG, globals(), locals()) + except Exception as error: + logging.exception(error) + raise error diff --git a/compass-deck/utils/daemonize.py b/compass-deck/utils/daemonize.py new file mode 100644 index 0000000..f02bfb9 --- /dev/null +++ b/compass-deck/utils/daemonize.py @@ -0,0 +1,76 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to provider util functions in all compass code + + .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com> +""" +import daemon +import logging +import signal +import sys +import time + +from compass.utils import flags + + +flags.add_bool('daemonize', + help='run as daemon', + default=False) + + +BUSY = False +KILLED = False + + +def handle_term(signum, frame): + """Handle sig term.""" + global KILLED + logging.info('Caught signal %s in %s', signum, frame) + KILLED = True + if not BUSY: + sys.exit(0) + + +def _daemon(callback, run_interval): + """help function to run callback in daemon.""" + global BUSY + signal.signal(signal.SIGTERM, handle_term) + signal.signal(signal.SIGHUP, handle_term) + + while True: + BUSY = True + callback() + BUSY = False + if KILLED: + logging.info('exit loop') + break + + if run_interval > 0: + logging.info('will rerun after %s seconds', + flags.OPTIONS.run_interval) + time.sleep(flags.OPTIONS.run_interval) + else: + logging.info('finish loop') + break + + +def daemonize(callback, run_interval, **kwargs): + """daemonize callback and run every run_interval seconds.""" + if flags.OPTIONS.daemonize: + with daemon.DaemonContext(**kwargs): + logging.info('run as daemon') + _daemon(callback, run_interval) + else: + _daemon(callback, run_interval) diff --git a/compass-deck/utils/flags.py b/compass-deck/utils/flags.py new file mode 100644 index 0000000..a3169f5 --- /dev/null +++ b/compass-deck/utils/flags.py @@ -0,0 +1,91 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to load flags. + + .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com> +""" +import sys + +from optparse import OptionParser + + +class Flags(object): + """Class to store flags.""" + + PARSER = OptionParser() + PARSED_OPTIONS = None + + @classmethod + def parse_args(cls): + """parse args.""" + (options, argv) = Flags.PARSER.parse_args() + sys.argv = [sys.argv[0]] + argv + Flags.PARSED_OPTIONS = options + + def __getattr__(self, name): + if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): + return getattr(Flags.PARSED_OPTIONS, name) + + for option in Flags.PARSER.option_list: + if option.dest == name: + return option.default + + raise AttributeError('Option instance has no attribute %s' % name) + + def __setattr__(self, name, value): + if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): + setattr(Flags.PARSED_OPTIONS, name, value) + return + + for option in Flags.PARSER.option_list: + if option.dest == name: + option.default = value + return + + object.__setattr__(self, name, value) + + +OPTIONS = Flags() + + +def init(): + """Init flag parsing.""" + OPTIONS.parse_args() + + +def add(flagname, **kwargs): + """Add a flag name and its setting. + + :param flagname: flag name declared in cmd as --<flagname>=... + :type flagname: str + """ + Flags.PARSER.add_option('--%s' % flagname, + dest=flagname, **kwargs) + + +def add_bool(flagname, default=True, **kwargs): + """Add a bool flag name and its setting. + + :param flagname: flag name declared in cmd as --[no]<flagname>. + :type flagname: str + :param default: default value + :type default: bool + """ + Flags.PARSER.add_option('--%s' % flagname, + dest=flagname, default=default, + action="store_true", **kwargs) + Flags.PARSER.add_option('--no%s' % flagname, + dest=flagname, + action="store_false", **kwargs) diff --git a/compass-deck/utils/logsetting.py b/compass-deck/utils/logsetting.py new file mode 100644 index 0000000..836ebcb --- /dev/null +++ b/compass-deck/utils/logsetting.py @@ -0,0 +1,108 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to setup logging configuration. + + .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com> +""" + +import logging +import logging.handlers +import os +import os.path +import sys + +from compass.utils import flags +from compass.utils import setting_wrapper as setting + + +flags.add('loglevel', + help='logging level', default=setting.DEFAULT_LOGLEVEL) +flags.add('logdir', + help='logging directory', default=setting.DEFAULT_LOGDIR) +flags.add('logfile', + help='logging filename', default=None) +flags.add('log_interval', type='int', + help='log interval', default=setting.DEFAULT_LOGINTERVAL) +flags.add('log_interval_unit', + help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT) +flags.add('log_format', + help='log format', default=setting.DEFAULT_LOGFORMAT) +flags.add('log_backup_count', type='int', + help='log backup count', default=setting.DEFAULT_LOGBACKUPCOUNT) + + +# mapping str setting in flag --loglevel to logging level. +LOGLEVEL_MAPPING = { + 'finest': logging.DEBUG - 2, # more detailed log. + 'fine': logging.DEBUG - 1, # detailed log. + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, +} + + +logging.addLevelName(LOGLEVEL_MAPPING['fine'], 'fine') +logging.addLevelName(LOGLEVEL_MAPPING['finest'], 'finest') + + +# disable logging when logsetting.init not called +logging.getLogger().setLevel(logging.CRITICAL) + + +def getLevelByName(level_name): + """Get log level by level name.""" + return LOGLEVEL_MAPPING[level_name] + + +def init(): + """Init loggsetting. It should be called after flags.init.""" + loglevel = flags.OPTIONS.loglevel.lower() + logdir = flags.OPTIONS.logdir + logfile = flags.OPTIONS.logfile + logger = logging.getLogger() + if logger.handlers: + for handler in logger.handlers: + logger.removeHandler(handler) + + if logdir: + if not logfile: + logfile = '%s.log' % os.path.basename(sys.argv[0]) + + handler = logging.handlers.TimedRotatingFileHandler( + os.path.join(logdir, logfile), + when=flags.OPTIONS.log_interval_unit, + interval=flags.OPTIONS.log_interval, + backupCount=flags.OPTIONS.log_backup_count) + else: + if not logfile: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.handlers.TimedRotatingFileHandler( + logfile, + when=flags.OPTIONS.log_interval_unit, + interval=flags.OPTIONS.log_interval, + backupCount=flags.OPTIONS.log_backup_count) + + if loglevel in LOGLEVEL_MAPPING: + logger.setLevel(LOGLEVEL_MAPPING[loglevel]) + handler.setLevel(LOGLEVEL_MAPPING[loglevel]) + + formatter = logging.Formatter( + flags.OPTIONS.log_format) + + handler.setFormatter(formatter) + logger.addHandler(handler) diff --git a/compass-deck/utils/setting_wrapper.py b/compass-deck/utils/setting_wrapper.py new file mode 100644 index 0000000..0b3e9f7 --- /dev/null +++ b/compass-deck/utils/setting_wrapper.py @@ -0,0 +1,175 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""comapss setting wrapper. + + .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> +""" +import datetime +import lazypy +import logging +import os +import os.path + + +# default setting +CONFIG_DIR = os.environ.get('COMPASS_CONFIG_DIR', '/etc/compass') +SQLALCHEMY_DATABASE_URI = 'sqlite://' +SQLALCHEMY_DATABASE_POOL_TYPE = 'static' +COBBLER_INSTALLATION_LOGDIR = '/var/log/cobbler/anamon' +CHEF_INSTALLATION_LOGDIR = '/var/log/chef' +INSTALLATION_LOGDIR = { + 'CobblerInstaller': COBBLER_INSTALLATION_LOGDIR, + 'ChefInstaller': CHEF_INSTALLATION_LOGDIR +} +CLUSTERHOST_INATALLATION_LOGDIR_NAME = 'name' +HOST_INSTALLATION_LOGDIR_NAME = 'name' +DEFAULT_LOGLEVEL = 'debug' +DEFAULT_LOGDIR = '/tmp' +DEFAULT_LOGINTERVAL = 1 +DEFAULT_LOGINTERVAL_UNIT = 'h' +DEFAULT_LOGFORMAT = ( + '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s') +DEFAULT_LOGBACKUPCOUNT = 5 +WEB_LOGFILE = '' +CELERY_LOGFILE = '' +CELERYCONFIG_DIR = lazypy.delay(lambda: CONFIG_DIR) +CELERYCONFIG_FILE = '' +PROGRESS_UPDATE_INTERVAL = 30 +POLLSWITCH_INTERVAL = 60 +SWITCHES = [ +] + +USER_AUTH_HEADER_NAME = 'X-Auth-Token' +USER_TOKEN_DURATION = '2h' +COMPASS_ADMIN_EMAIL = 'admin@huawei.com' +COMPASS_ADMIN_PASSWORD = 'admin' +COMPASS_DEFAULT_PERMISSIONS = [ + 'list_permissions', +] +SWITCHES_DEFAULT_FILTERS = [] +DEFAULT_SWITCH_IP = '0.0.0.0' +DEFAULT_SWITCH_PORT = 0 + +COMPASS_SUPPORTED_PROXY = 'http://127.0.0.1:3128' +COMPASS_SUPPORTED_DEFAULT_NOPROXY = ['127.0.0.1'] +COMPASS_SUPPORTED_NTP_SERVER = '127.0.0.1' +COMPASS_SUPPORTED_DNS_SERVERS = ['127.0.0.1'] +COMPASS_SUPPORTED_DOMAINS = [] +COMPASS_SUPPORTED_DEFAULT_GATEWAY = '127.0.0.1' +COMPASS_SUPPORTED_LOCAL_REPO = 'http://127.0.0.1' + +PROGRESS_UPDATE_PID_FILE = '/var/run/progress_update.pid' + +PROXY_URL_PREFIX = 'http://10.145.81.205:5000' + +OS_INSTALLER_DIR = '' +PACKAGE_INSTALLER_DIR = '' +OS_DIR = '' +ADAPTER_DIR = '' +OS_METADATA_DIR = '' +PACKAGE_METADATA_DIR = '' +FLAVOR_METADATA_DIR = '' +OS_FIELD_DIR = '' +PACKAGE_FIELD_DIR = '' +FLAVOR_FIELD_DIR = '' +ADAPTER_ROLE_DIR = '' +ADAPTER_FLAVOR_DIR = '' +VALIDATOR_DIR = '' +CALLBACK_DIR = '' +TMPL_DIR = '' +MACHINE_LIST_DIR = '' +PROGRESS_CALCULATOR_DIR = '' +OS_MAPPING_DIR = '' +FLAVOR_MAPPING_DIR = '' +PLUGINS_DIR = '' + +if ( + 'COMPASS_IGNORE_SETTING' in os.environ and + os.environ['COMPASS_IGNORE_SETTING'] +): + pass +else: + if 'COMPASS_SETTING' in os.environ: + SETTING = os.environ['COMPASS_SETTING'] + else: + SETTING = '/etc/compass/setting' + + try: + logging.info('load setting from %s', SETTING) + execfile(SETTING, globals(), locals()) + except Exception as error: + logging.exception(error) + raise error + +if not OS_INSTALLER_DIR: + OS_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'os_installer') + +if not PACKAGE_INSTALLER_DIR: + PACKAGE_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'package_installer') + +if not OS_DIR: + OS_DIR = os.path.join(CONFIG_DIR, 'os') + +if not ADAPTER_DIR: + ADAPTER_DIR = os.path.join(CONFIG_DIR, 'adapter') + +if not OS_METADATA_DIR: + OS_METADATA_DIR = os.path.join(CONFIG_DIR, 'os_metadata') + +if not PACKAGE_METADATA_DIR: + PACKAGE_METADATA_DIR = os.path.join(CONFIG_DIR, 'package_metadata') + +if not FLAVOR_METADATA_DIR: + FLAVOR_METADATA_DIR = os.path.join(CONFIG_DIR, 'flavor_metadata') + +if not OS_FIELD_DIR: + OS_FIELD_DIR = os.path.join(CONFIG_DIR, 'os_field') + +if not PACKAGE_FIELD_DIR: + PACKAGE_FIELD_DIR = os.path.join(CONFIG_DIR, 'package_field') + +if not FLAVOR_FIELD_DIR: + FLAVOR_FIELD_DIR = os.path.join(CONFIG_DIR, 'flavor_field') + +if not ADAPTER_ROLE_DIR: + ADAPTER_ROLE_DIR = os.path.join(CONFIG_DIR, 'role') + +if not ADAPTER_FLAVOR_DIR: + ADAPTER_FLAVOR_DIR = os.path.join(CONFIG_DIR, 'flavor') + +if not VALIDATOR_DIR: + VALIDATOR_DIR = os.path.join(CONFIG_DIR, 'validator') + +if not CALLBACK_DIR: + CALLBACK_DIR = os.path.join(CONFIG_DIR, 'callback') + +if not TMPL_DIR: + TMPL_DIR = os.path.join(CONFIG_DIR, 'templates') + +if not MACHINE_LIST_DIR: + MACHINE_LIST_DIR = os.path.join(CONFIG_DIR, 'machine_list') + +if not PROGRESS_CALCULATOR_DIR: + PROGRESS_CALCULATOR_DIR = os.path.join(CONFIG_DIR, 'progress_calculator') + +if not OS_MAPPING_DIR: + OS_MAPPING_DIR = os.path.join(CONFIG_DIR, 'os_mapping') + +if not FLAVOR_MAPPING_DIR: + FLAVOR_MAPPING_DIR = os.path.join(CONFIG_DIR, 'flavor_mapping') + +if not PLUGINS_DIR: + PLUGINS_DIR = os.environ.get('COMPASS_PLUGINS_DIR', + os.path.join(CONFIG_DIR, 'plugins')) diff --git a/compass-deck/utils/util.py b/compass-deck/utils/util.py new file mode 100644 index 0000000..39978ca --- /dev/null +++ b/compass-deck/utils/util.py @@ -0,0 +1,395 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to provider util functions in all compass code + + .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com> +""" + +import crypt +import datetime +import logging +import os +import os.path +import re +import setting_wrapper as setting +import sys +import warnings + + +def deprecated(func): + """This is a decorator which can be used to mark functions as deprecated. + + It will result in a warning being emitted when the function is used. + """ + def new_func(*args, **kwargs): + warnings.warn( + "Call to deprecated function %s." % func.__name__, + category=DeprecationWarning + ) + return func(*args, **kwargs) + + new_func.__name__ = func.__name__ + new_func.__doc__ = func.__doc__ + new_func.__dict__.update(func.__dict__) + return new_func + + +def parse_datetime(date_time, exception_class=Exception): + """Parse datetime str to get datetime object. + + The date time format is %Y-%m-%d %H:%M:%S + """ + try: + return datetime.datetime.strptime( + date_time, '%Y-%m-%d %H:%M:%S' + ) + except Exception as error: + logging.exception(error) + raise exception_class( + 'date time %s format is invalid' % date_time + ) + + +def parse_datetime_range(date_time_range, exception_class=Exception): + """parse datetime range str to pair of datetime objects. + + The date time range format is %Y-%m-%d %H:%M:%S,%Y-%m-%d %H:%M:%S + """ + try: + start, end = date_time_range.split(',') + except Exception as error: + logging.exception(error) + raise exception_class( + 'there is no `,` in date time range %s' % date_time_range + ) + if start: + start_datetime = parse_datetime(start, exception_class) + else: + start_datetime = None + if end: + end_datetime = parse_datetime(end, exception_class) + else: + end_datetime = None + return start_datetime, end_datetime + + +def parse_request_arg_dict(arg, exception_class=Exception): + """parse string to dict. + + The str is formatted like a=b;c=d and parsed to + {'a': 'b', 'c': 'd'} + """ + arg_dict = {} + arg_pairs = arg.split(';') + for arg_pair in arg_pairs: + try: + arg_name, arg_value = arg_pair.split('=', 1) + except Exception as error: + logging.exception(error) + raise exception_class( + 'there is no `=` in %s' % arg_pair + ) + arg_dict[arg_name] = arg_value + return arg_dict + + +def format_datetime(date_time): + """Generate string from datetime object.""" + return date_time.strftime("%Y-%m-%d %H:%M:%S") + + +def merge_dict(lhs, rhs, override=True): + """Merge nested right dict into left nested dict recursively. + + :param lhs: dict to be merged into. + :type lhs: dict + :param rhs: dict to merge from. + :type rhs: dict + :param override: the value in rhs overide the value in left if True. + :type override: boolean + """ + if not isinstance(lhs, dict) or not isinstance(rhs, dict): + if override: + return rhs + else: + return lhs + + for key, value in rhs.items(): + if key not in lhs: + lhs[key] = rhs[key] + else: + lhs[key] = merge_dict(lhs[key], value, override) + + return lhs + + +def recursive_merge_dict(name, all_dicts, parents): + """Recursively merge parent dict into base dict.""" + parent_name = parents.get(name, None) + base_dict = all_dicts.get(name, {}) + if not parent_name: + return base_dict + merged = recursive_merge_dict(parent_name, all_dicts, parents) + return merge_dict(base_dict, merged, override=False) + + +def encrypt(value, crypt_method=None): + """Get encrypted value.""" + if not crypt_method: + if hasattr(crypt, 'METHOD_MD5'): + crypt_method = crypt.METHOD_MD5 + else: + # for python2.7, copy python2.6 METHOD_MD5 logic here. + from random import choice + import string + + _saltchars = string.ascii_letters + string.digits + './' + + def _mksalt(): + """generate salt.""" + salt = '$1$' + salt += ''.join(choice(_saltchars) for _ in range(8)) + return salt + + crypt_method = _mksalt() + + return crypt.crypt(value, crypt_method) + + +def parse_time_interval(time_interval_str): + """parse string of time interval to time interval. + + supported time interval unit: ['d', 'w', 'h', 'm', 's'] + Examples: + time_interval_str: '3d 2h' time interval to 3 days and 2 hours. + """ + if not time_interval_str: + return 0 + + time_interval_tuple = [ + time_interval_element + for time_interval_element in time_interval_str.split(' ') + if time_interval_element + ] + time_interval_dict = {} + time_interval_unit_mapping = { + 'd': 'days', + 'w': 'weeks', + 'h': 'hours', + 'm': 'minutes', + 's': 'seconds' + } + for time_interval_element in time_interval_tuple: + mat = re.match(r'^([+-]?\d+)(w|d|h|m|s).*', time_interval_element) + if not mat: + continue + + time_interval_value = int(mat.group(1)) + time_interval_unit = time_interval_unit_mapping[mat.group(2)] + time_interval_dict[time_interval_unit] = ( + time_interval_dict.get(time_interval_unit, 0) + time_interval_value + ) + + time_interval = datetime.timedelta(**time_interval_dict) + if sys.version_info[0:2] > (2, 6): + return time_interval.total_seconds() + else: + return ( + time_interval.microseconds + ( + time_interval.seconds + time_interval.days * 24 * 3600 + ) * 1e6 + ) / 1e6 + + +def get_plugins_config_files(name, suffix=".conf"): + """walk through each of plugin to find all the config files in the""" + """name directory""" + + plugins_path = setting.PLUGINS_DIR + files = [] + if os.path.exists(plugins_path): + for plugin in os.listdir(plugins_path): + plugin_path = os.path.join(plugins_path, plugin) + plugin_config = os.path.join(plugin_path, name) + if os.path.exists(plugin_config): + for component in os.listdir(plugin_config): + if not component.endswith(suffix): + continue + files.append(os.path.join(plugin_config, component)) + return files + + +def load_configs( + config_dir, config_name_suffix='.conf', + env_globals={}, env_locals={} +): + """Load configurations from config dir.""" + """The config file could be in the config_dir or in plugins config_dir""" + """The plugins config_dir is formed as, for example /etc/compass/adapter""" + """Then the plugins config_dir is /etc/compass/plugins/xxx/adapter""" + + # TODO(Carl) instead of using config_dir, it should use a name such as + # adapter etc, however, doing it requires a lot client sites changes, + # will do it later. + + configs = [] + config_files = [] + config_dir = str(config_dir) + + """search for config_dir""" + if os.path.exists(config_dir): + for component in os.listdir(config_dir): + if not component.endswith(config_name_suffix): + continue + config_files.append(os.path.join(config_dir, component)) + + """search for plugins config_dir""" + index = config_dir.rfind("/") + + config_files.extend(get_plugins_config_files(config_dir[index + 1:], + config_name_suffix)) + + if not config_files: + logging.error('path %s and plugins does not exist', config_dir) + for path in config_files: + logging.debug('load config from %s', path) + config_globals = {} + config_globals.update(env_globals) + config_locals = {} + config_locals.update(env_locals) + try: + execfile(path, config_globals, config_locals) + except Exception as error: + logging.exception(error) + raise error + configs.append(config_locals) + return configs + + +def pretty_print(*contents): + """pretty print contents.""" + if len(contents) == 0: + print "" + else: + print "\n".join(content for content in contents) + + +def get_switch_machines_from_file(filename): + """get switch machines from file.""" + switches = [] + switch_machines = {} + with open(filename) as switch_file: + for line in switch_file: + line = line.strip() + if not line: + # ignore empty line + continue + + if line.startswith('#'): + # ignore comments + continue + + columns = [column for column in line.split(',')] + if not columns: + # ignore empty line + continue + + if columns[0] == 'switch': + (switch_ip, switch_vendor, switch_version, + switch_community, switch_state) = columns[1:] + switches.append({ + 'ip': switch_ip, + 'vendor': switch_vendor, + 'credentials': { + 'version': switch_version, + 'community': switch_community, + }, + 'state': switch_state, + }) + elif columns[0] == 'machine': + switch_ip, switch_port, mac = columns[1:] + switch_machines.setdefault(switch_ip, []).append({ + 'mac': mac, + 'port': switch_port, + }) + + return (switches, switch_machines) + + +def execute_cli_by_ssh(cmd, host, username, password=None, + keyfile='/root/.ssh/id_rsa', nowait=False): + """SSH to execute script on remote machine + + :param host: ip of the remote machine + :param username: username to access the remote machine + :param password: password to access the remote machine + :param cmd: command to execute + + """ + if not cmd: + logging.error("No command found!") + raise Exception('No command found!') + + if nowait: + cmd = "nohup %s >/dev/null 2>&1 &" % cmd + + stdin = None + stdout = None + stderr = None + try: + import paramiko + from paramiko import ssh_exception + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + if password: + client.connect(host, username=username, password=password) + else: + client.load_system_host_keys() + client.connect( + host, username=username, + key_filename=keyfile, look_for_keys=True + ) + stdin, stdout, stderr = client.exec_command(cmd) + result = stdout.readlines() + logging.info("result of command '%s' is '%s'!" % (cmd, result)) + return result + + except ImportError: + err_msg = "Cannot find Paramiko package!" + logging.error(err_msg) + raise ImportError(err_msg) + + except (ssh_exception.BadHostKeyException, + ssh_exception.AuthenticationException, + ssh_exception.SSHException): + + err_msg = 'SSH connection error or command execution failed!' + logging.error(err_msg) + raise Exception(err_msg) + + except Exception as exc: + logging.error( + 'Failed to execute command "%s", exception is %s' % (cmd, exc) + ) + raise Exception(exc) + + finally: + for resource in [stdin, stdout, stderr]: + if resource: + resource.close() + + client.close() |