summaryrefslogtreecommitdiffstats
path: root/compass-deck/utils
diff options
context:
space:
mode:
authorHarry Huang <huangxiangyu5@huawei.com>2017-11-01 11:56:50 +0800
committerHarry Huang <huangxiangyu5@huawei.com>2017-11-03 11:48:46 +0800
commit905b0231e93ce2409a45dd6c4f5f983689fdb790 (patch)
tree8b3c8c78773194e048072368fe793135a05e44f1 /compass-deck/utils
parent3656ab7b5e3f2f26f7c98f9dcc97b3c461fa2a76 (diff)
Add compass-deck
RESTful API and DB Handlers for Compass Change-Id: I1ce411f279943764c286ea48dca9185d453cf254 Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>
Diffstat (limited to 'compass-deck/utils')
-rw-r--r--compass-deck/utils/__init__.py13
-rw-r--r--compass-deck/utils/celeryconfig_wrapper.py44
-rw-r--r--compass-deck/utils/daemonize.py76
-rw-r--r--compass-deck/utils/flags.py91
-rw-r--r--compass-deck/utils/logsetting.py108
-rw-r--r--compass-deck/utils/setting_wrapper.py175
-rw-r--r--compass-deck/utils/util.py395
7 files changed, 902 insertions, 0 deletions
diff --git a/compass-deck/utils/__init__.py b/compass-deck/utils/__init__.py
new file mode 100644
index 0000000..4ee55a4
--- /dev/null
+++ b/compass-deck/utils/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-deck/utils/celeryconfig_wrapper.py b/compass-deck/utils/celeryconfig_wrapper.py
new file mode 100644
index 0000000..b6644ba
--- /dev/null
+++ b/compass-deck/utils/celeryconfig_wrapper.py
@@ -0,0 +1,44 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""celeryconfig wrapper.
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+import os.path
+import urllib
+
+from compass.utils import setting_wrapper as setting
+
+
+# CELERY_RESULT_BACKEND = 'amqp://'
+
+# BROKER_URL = 'amqp://guest:guest@localhost:5672//'
+
+
+CELERY_IMPORTS = ('compass.tasks.tasks',)
+
+
+if setting.CELERYCONFIG_FILE:
+ CELERY_CONFIG = os.path.join(
+ str(setting.CELERYCONFIG_DIR),
+ str(setting.CELERYCONFIG_FILE))
+
+ try:
+ logging.info('load celery config from %s', CELERY_CONFIG)
+ execfile(CELERY_CONFIG, globals(), locals())
+ except Exception as error:
+ logging.exception(error)
+ raise error
diff --git a/compass-deck/utils/daemonize.py b/compass-deck/utils/daemonize.py
new file mode 100644
index 0000000..f02bfb9
--- /dev/null
+++ b/compass-deck/utils/daemonize.py
@@ -0,0 +1,76 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to provider util functions in all compass code
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import daemon
+import logging
+import signal
+import sys
+import time
+
+from compass.utils import flags
+
+
+flags.add_bool('daemonize',
+ help='run as daemon',
+ default=False)
+
+
+BUSY = False
+KILLED = False
+
+
+def handle_term(signum, frame):
+ """Handle sig term."""
+ global KILLED
+ logging.info('Caught signal %s in %s', signum, frame)
+ KILLED = True
+ if not BUSY:
+ sys.exit(0)
+
+
+def _daemon(callback, run_interval):
+ """help function to run callback in daemon."""
+ global BUSY
+ signal.signal(signal.SIGTERM, handle_term)
+ signal.signal(signal.SIGHUP, handle_term)
+
+ while True:
+ BUSY = True
+ callback()
+ BUSY = False
+ if KILLED:
+ logging.info('exit loop')
+ break
+
+ if run_interval > 0:
+ logging.info('will rerun after %s seconds',
+ flags.OPTIONS.run_interval)
+ time.sleep(flags.OPTIONS.run_interval)
+ else:
+ logging.info('finish loop')
+ break
+
+
+def daemonize(callback, run_interval, **kwargs):
+ """daemonize callback and run every run_interval seconds."""
+ if flags.OPTIONS.daemonize:
+ with daemon.DaemonContext(**kwargs):
+ logging.info('run as daemon')
+ _daemon(callback, run_interval)
+ else:
+ _daemon(callback, run_interval)
diff --git a/compass-deck/utils/flags.py b/compass-deck/utils/flags.py
new file mode 100644
index 0000000..a3169f5
--- /dev/null
+++ b/compass-deck/utils/flags.py
@@ -0,0 +1,91 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to load flags.
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import sys
+
+from optparse import OptionParser
+
+
+class Flags(object):
+ """Class to store flags."""
+
+ PARSER = OptionParser()
+ PARSED_OPTIONS = None
+
+ @classmethod
+ def parse_args(cls):
+ """parse args."""
+ (options, argv) = Flags.PARSER.parse_args()
+ sys.argv = [sys.argv[0]] + argv
+ Flags.PARSED_OPTIONS = options
+
+ def __getattr__(self, name):
+ if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name):
+ return getattr(Flags.PARSED_OPTIONS, name)
+
+ for option in Flags.PARSER.option_list:
+ if option.dest == name:
+ return option.default
+
+ raise AttributeError('Option instance has no attribute %s' % name)
+
+ def __setattr__(self, name, value):
+ if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name):
+ setattr(Flags.PARSED_OPTIONS, name, value)
+ return
+
+ for option in Flags.PARSER.option_list:
+ if option.dest == name:
+ option.default = value
+ return
+
+ object.__setattr__(self, name, value)
+
+
+OPTIONS = Flags()
+
+
+def init():
+ """Init flag parsing."""
+ OPTIONS.parse_args()
+
+
+def add(flagname, **kwargs):
+ """Add a flag name and its setting.
+
+ :param flagname: flag name declared in cmd as --<flagname>=...
+ :type flagname: str
+ """
+ Flags.PARSER.add_option('--%s' % flagname,
+ dest=flagname, **kwargs)
+
+
+def add_bool(flagname, default=True, **kwargs):
+ """Add a bool flag name and its setting.
+
+ :param flagname: flag name declared in cmd as --[no]<flagname>.
+ :type flagname: str
+ :param default: default value
+ :type default: bool
+ """
+ Flags.PARSER.add_option('--%s' % flagname,
+ dest=flagname, default=default,
+ action="store_true", **kwargs)
+ Flags.PARSER.add_option('--no%s' % flagname,
+ dest=flagname,
+ action="store_false", **kwargs)
diff --git a/compass-deck/utils/logsetting.py b/compass-deck/utils/logsetting.py
new file mode 100644
index 0000000..836ebcb
--- /dev/null
+++ b/compass-deck/utils/logsetting.py
@@ -0,0 +1,108 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to setup logging configuration.
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+
+import logging
+import logging.handlers
+import os
+import os.path
+import sys
+
+from compass.utils import flags
+from compass.utils import setting_wrapper as setting
+
+
+flags.add('loglevel',
+ help='logging level', default=setting.DEFAULT_LOGLEVEL)
+flags.add('logdir',
+ help='logging directory', default=setting.DEFAULT_LOGDIR)
+flags.add('logfile',
+ help='logging filename', default=None)
+flags.add('log_interval', type='int',
+ help='log interval', default=setting.DEFAULT_LOGINTERVAL)
+flags.add('log_interval_unit',
+ help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT)
+flags.add('log_format',
+ help='log format', default=setting.DEFAULT_LOGFORMAT)
+flags.add('log_backup_count', type='int',
+ help='log backup count', default=setting.DEFAULT_LOGBACKUPCOUNT)
+
+
+# mapping str setting in flag --loglevel to logging level.
+LOGLEVEL_MAPPING = {
+ 'finest': logging.DEBUG - 2, # more detailed log.
+ 'fine': logging.DEBUG - 1, # detailed log.
+ 'debug': logging.DEBUG,
+ 'info': logging.INFO,
+ 'warning': logging.WARNING,
+ 'error': logging.ERROR,
+ 'critical': logging.CRITICAL,
+}
+
+
+logging.addLevelName(LOGLEVEL_MAPPING['fine'], 'fine')
+logging.addLevelName(LOGLEVEL_MAPPING['finest'], 'finest')
+
+
+# disable logging when logsetting.init not called
+logging.getLogger().setLevel(logging.CRITICAL)
+
+
+def getLevelByName(level_name):
+ """Get log level by level name."""
+ return LOGLEVEL_MAPPING[level_name]
+
+
+def init():
+ """Init loggsetting. It should be called after flags.init."""
+ loglevel = flags.OPTIONS.loglevel.lower()
+ logdir = flags.OPTIONS.logdir
+ logfile = flags.OPTIONS.logfile
+ logger = logging.getLogger()
+ if logger.handlers:
+ for handler in logger.handlers:
+ logger.removeHandler(handler)
+
+ if logdir:
+ if not logfile:
+ logfile = '%s.log' % os.path.basename(sys.argv[0])
+
+ handler = logging.handlers.TimedRotatingFileHandler(
+ os.path.join(logdir, logfile),
+ when=flags.OPTIONS.log_interval_unit,
+ interval=flags.OPTIONS.log_interval,
+ backupCount=flags.OPTIONS.log_backup_count)
+ else:
+ if not logfile:
+ handler = logging.StreamHandler(sys.stderr)
+ else:
+ handler = logging.handlers.TimedRotatingFileHandler(
+ logfile,
+ when=flags.OPTIONS.log_interval_unit,
+ interval=flags.OPTIONS.log_interval,
+ backupCount=flags.OPTIONS.log_backup_count)
+
+ if loglevel in LOGLEVEL_MAPPING:
+ logger.setLevel(LOGLEVEL_MAPPING[loglevel])
+ handler.setLevel(LOGLEVEL_MAPPING[loglevel])
+
+ formatter = logging.Formatter(
+ flags.OPTIONS.log_format)
+
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
diff --git a/compass-deck/utils/setting_wrapper.py b/compass-deck/utils/setting_wrapper.py
new file mode 100644
index 0000000..0b3e9f7
--- /dev/null
+++ b/compass-deck/utils/setting_wrapper.py
@@ -0,0 +1,175 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""comapss setting wrapper.
+
+ .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com>
+"""
+import datetime
+import lazypy
+import logging
+import os
+import os.path
+
+
+# default setting
+CONFIG_DIR = os.environ.get('COMPASS_CONFIG_DIR', '/etc/compass')
+SQLALCHEMY_DATABASE_URI = 'sqlite://'
+SQLALCHEMY_DATABASE_POOL_TYPE = 'static'
+COBBLER_INSTALLATION_LOGDIR = '/var/log/cobbler/anamon'
+CHEF_INSTALLATION_LOGDIR = '/var/log/chef'
+INSTALLATION_LOGDIR = {
+ 'CobblerInstaller': COBBLER_INSTALLATION_LOGDIR,
+ 'ChefInstaller': CHEF_INSTALLATION_LOGDIR
+}
+CLUSTERHOST_INATALLATION_LOGDIR_NAME = 'name'
+HOST_INSTALLATION_LOGDIR_NAME = 'name'
+DEFAULT_LOGLEVEL = 'debug'
+DEFAULT_LOGDIR = '/tmp'
+DEFAULT_LOGINTERVAL = 1
+DEFAULT_LOGINTERVAL_UNIT = 'h'
+DEFAULT_LOGFORMAT = (
+ '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s')
+DEFAULT_LOGBACKUPCOUNT = 5
+WEB_LOGFILE = ''
+CELERY_LOGFILE = ''
+CELERYCONFIG_DIR = lazypy.delay(lambda: CONFIG_DIR)
+CELERYCONFIG_FILE = ''
+PROGRESS_UPDATE_INTERVAL = 30
+POLLSWITCH_INTERVAL = 60
+SWITCHES = [
+]
+
+USER_AUTH_HEADER_NAME = 'X-Auth-Token'
+USER_TOKEN_DURATION = '2h'
+COMPASS_ADMIN_EMAIL = 'admin@huawei.com'
+COMPASS_ADMIN_PASSWORD = 'admin'
+COMPASS_DEFAULT_PERMISSIONS = [
+ 'list_permissions',
+]
+SWITCHES_DEFAULT_FILTERS = []
+DEFAULT_SWITCH_IP = '0.0.0.0'
+DEFAULT_SWITCH_PORT = 0
+
+COMPASS_SUPPORTED_PROXY = 'http://127.0.0.1:3128'
+COMPASS_SUPPORTED_DEFAULT_NOPROXY = ['127.0.0.1']
+COMPASS_SUPPORTED_NTP_SERVER = '127.0.0.1'
+COMPASS_SUPPORTED_DNS_SERVERS = ['127.0.0.1']
+COMPASS_SUPPORTED_DOMAINS = []
+COMPASS_SUPPORTED_DEFAULT_GATEWAY = '127.0.0.1'
+COMPASS_SUPPORTED_LOCAL_REPO = 'http://127.0.0.1'
+
+PROGRESS_UPDATE_PID_FILE = '/var/run/progress_update.pid'
+
+PROXY_URL_PREFIX = 'http://10.145.81.205:5000'
+
+OS_INSTALLER_DIR = ''
+PACKAGE_INSTALLER_DIR = ''
+OS_DIR = ''
+ADAPTER_DIR = ''
+OS_METADATA_DIR = ''
+PACKAGE_METADATA_DIR = ''
+FLAVOR_METADATA_DIR = ''
+OS_FIELD_DIR = ''
+PACKAGE_FIELD_DIR = ''
+FLAVOR_FIELD_DIR = ''
+ADAPTER_ROLE_DIR = ''
+ADAPTER_FLAVOR_DIR = ''
+VALIDATOR_DIR = ''
+CALLBACK_DIR = ''
+TMPL_DIR = ''
+MACHINE_LIST_DIR = ''
+PROGRESS_CALCULATOR_DIR = ''
+OS_MAPPING_DIR = ''
+FLAVOR_MAPPING_DIR = ''
+PLUGINS_DIR = ''
+
+if (
+ 'COMPASS_IGNORE_SETTING' in os.environ and
+ os.environ['COMPASS_IGNORE_SETTING']
+):
+ pass
+else:
+ if 'COMPASS_SETTING' in os.environ:
+ SETTING = os.environ['COMPASS_SETTING']
+ else:
+ SETTING = '/etc/compass/setting'
+
+ try:
+ logging.info('load setting from %s', SETTING)
+ execfile(SETTING, globals(), locals())
+ except Exception as error:
+ logging.exception(error)
+ raise error
+
+if not OS_INSTALLER_DIR:
+ OS_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'os_installer')
+
+if not PACKAGE_INSTALLER_DIR:
+ PACKAGE_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'package_installer')
+
+if not OS_DIR:
+ OS_DIR = os.path.join(CONFIG_DIR, 'os')
+
+if not ADAPTER_DIR:
+ ADAPTER_DIR = os.path.join(CONFIG_DIR, 'adapter')
+
+if not OS_METADATA_DIR:
+ OS_METADATA_DIR = os.path.join(CONFIG_DIR, 'os_metadata')
+
+if not PACKAGE_METADATA_DIR:
+ PACKAGE_METADATA_DIR = os.path.join(CONFIG_DIR, 'package_metadata')
+
+if not FLAVOR_METADATA_DIR:
+ FLAVOR_METADATA_DIR = os.path.join(CONFIG_DIR, 'flavor_metadata')
+
+if not OS_FIELD_DIR:
+ OS_FIELD_DIR = os.path.join(CONFIG_DIR, 'os_field')
+
+if not PACKAGE_FIELD_DIR:
+ PACKAGE_FIELD_DIR = os.path.join(CONFIG_DIR, 'package_field')
+
+if not FLAVOR_FIELD_DIR:
+ FLAVOR_FIELD_DIR = os.path.join(CONFIG_DIR, 'flavor_field')
+
+if not ADAPTER_ROLE_DIR:
+ ADAPTER_ROLE_DIR = os.path.join(CONFIG_DIR, 'role')
+
+if not ADAPTER_FLAVOR_DIR:
+ ADAPTER_FLAVOR_DIR = os.path.join(CONFIG_DIR, 'flavor')
+
+if not VALIDATOR_DIR:
+ VALIDATOR_DIR = os.path.join(CONFIG_DIR, 'validator')
+
+if not CALLBACK_DIR:
+ CALLBACK_DIR = os.path.join(CONFIG_DIR, 'callback')
+
+if not TMPL_DIR:
+ TMPL_DIR = os.path.join(CONFIG_DIR, 'templates')
+
+if not MACHINE_LIST_DIR:
+ MACHINE_LIST_DIR = os.path.join(CONFIG_DIR, 'machine_list')
+
+if not PROGRESS_CALCULATOR_DIR:
+ PROGRESS_CALCULATOR_DIR = os.path.join(CONFIG_DIR, 'progress_calculator')
+
+if not OS_MAPPING_DIR:
+ OS_MAPPING_DIR = os.path.join(CONFIG_DIR, 'os_mapping')
+
+if not FLAVOR_MAPPING_DIR:
+ FLAVOR_MAPPING_DIR = os.path.join(CONFIG_DIR, 'flavor_mapping')
+
+if not PLUGINS_DIR:
+ PLUGINS_DIR = os.environ.get('COMPASS_PLUGINS_DIR',
+ os.path.join(CONFIG_DIR, 'plugins'))
diff --git a/compass-deck/utils/util.py b/compass-deck/utils/util.py
new file mode 100644
index 0000000..39978ca
--- /dev/null
+++ b/compass-deck/utils/util.py
@@ -0,0 +1,395 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to provider util functions in all compass code
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+
+import crypt
+import datetime
+import logging
+import os
+import os.path
+import re
+import setting_wrapper as setting
+import sys
+import warnings
+
+
+def deprecated(func):
+ """This is a decorator which can be used to mark functions as deprecated.
+
+ It will result in a warning being emitted when the function is used.
+ """
+ def new_func(*args, **kwargs):
+ warnings.warn(
+ "Call to deprecated function %s." % func.__name__,
+ category=DeprecationWarning
+ )
+ return func(*args, **kwargs)
+
+ new_func.__name__ = func.__name__
+ new_func.__doc__ = func.__doc__
+ new_func.__dict__.update(func.__dict__)
+ return new_func
+
+
+def parse_datetime(date_time, exception_class=Exception):
+ """Parse datetime str to get datetime object.
+
+ The date time format is %Y-%m-%d %H:%M:%S
+ """
+ try:
+ return datetime.datetime.strptime(
+ date_time, '%Y-%m-%d %H:%M:%S'
+ )
+ except Exception as error:
+ logging.exception(error)
+ raise exception_class(
+ 'date time %s format is invalid' % date_time
+ )
+
+
+def parse_datetime_range(date_time_range, exception_class=Exception):
+ """parse datetime range str to pair of datetime objects.
+
+ The date time range format is %Y-%m-%d %H:%M:%S,%Y-%m-%d %H:%M:%S
+ """
+ try:
+ start, end = date_time_range.split(',')
+ except Exception as error:
+ logging.exception(error)
+ raise exception_class(
+ 'there is no `,` in date time range %s' % date_time_range
+ )
+ if start:
+ start_datetime = parse_datetime(start, exception_class)
+ else:
+ start_datetime = None
+ if end:
+ end_datetime = parse_datetime(end, exception_class)
+ else:
+ end_datetime = None
+ return start_datetime, end_datetime
+
+
+def parse_request_arg_dict(arg, exception_class=Exception):
+ """parse string to dict.
+
+ The str is formatted like a=b;c=d and parsed to
+ {'a': 'b', 'c': 'd'}
+ """
+ arg_dict = {}
+ arg_pairs = arg.split(';')
+ for arg_pair in arg_pairs:
+ try:
+ arg_name, arg_value = arg_pair.split('=', 1)
+ except Exception as error:
+ logging.exception(error)
+ raise exception_class(
+ 'there is no `=` in %s' % arg_pair
+ )
+ arg_dict[arg_name] = arg_value
+ return arg_dict
+
+
+def format_datetime(date_time):
+ """Generate string from datetime object."""
+ return date_time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def merge_dict(lhs, rhs, override=True):
+ """Merge nested right dict into left nested dict recursively.
+
+ :param lhs: dict to be merged into.
+ :type lhs: dict
+ :param rhs: dict to merge from.
+ :type rhs: dict
+ :param override: the value in rhs overide the value in left if True.
+ :type override: boolean
+ """
+ if not isinstance(lhs, dict) or not isinstance(rhs, dict):
+ if override:
+ return rhs
+ else:
+ return lhs
+
+ for key, value in rhs.items():
+ if key not in lhs:
+ lhs[key] = rhs[key]
+ else:
+ lhs[key] = merge_dict(lhs[key], value, override)
+
+ return lhs
+
+
+def recursive_merge_dict(name, all_dicts, parents):
+ """Recursively merge parent dict into base dict."""
+ parent_name = parents.get(name, None)
+ base_dict = all_dicts.get(name, {})
+ if not parent_name:
+ return base_dict
+ merged = recursive_merge_dict(parent_name, all_dicts, parents)
+ return merge_dict(base_dict, merged, override=False)
+
+
+def encrypt(value, crypt_method=None):
+ """Get encrypted value."""
+ if not crypt_method:
+ if hasattr(crypt, 'METHOD_MD5'):
+ crypt_method = crypt.METHOD_MD5
+ else:
+ # for python2.7, copy python2.6 METHOD_MD5 logic here.
+ from random import choice
+ import string
+
+ _saltchars = string.ascii_letters + string.digits + './'
+
+ def _mksalt():
+ """generate salt."""
+ salt = '$1$'
+ salt += ''.join(choice(_saltchars) for _ in range(8))
+ return salt
+
+ crypt_method = _mksalt()
+
+ return crypt.crypt(value, crypt_method)
+
+
+def parse_time_interval(time_interval_str):
+ """parse string of time interval to time interval.
+
+ supported time interval unit: ['d', 'w', 'h', 'm', 's']
+ Examples:
+ time_interval_str: '3d 2h' time interval to 3 days and 2 hours.
+ """
+ if not time_interval_str:
+ return 0
+
+ time_interval_tuple = [
+ time_interval_element
+ for time_interval_element in time_interval_str.split(' ')
+ if time_interval_element
+ ]
+ time_interval_dict = {}
+ time_interval_unit_mapping = {
+ 'd': 'days',
+ 'w': 'weeks',
+ 'h': 'hours',
+ 'm': 'minutes',
+ 's': 'seconds'
+ }
+ for time_interval_element in time_interval_tuple:
+ mat = re.match(r'^([+-]?\d+)(w|d|h|m|s).*', time_interval_element)
+ if not mat:
+ continue
+
+ time_interval_value = int(mat.group(1))
+ time_interval_unit = time_interval_unit_mapping[mat.group(2)]
+ time_interval_dict[time_interval_unit] = (
+ time_interval_dict.get(time_interval_unit, 0) + time_interval_value
+ )
+
+ time_interval = datetime.timedelta(**time_interval_dict)
+ if sys.version_info[0:2] > (2, 6):
+ return time_interval.total_seconds()
+ else:
+ return (
+ time_interval.microseconds + (
+ time_interval.seconds + time_interval.days * 24 * 3600
+ ) * 1e6
+ ) / 1e6
+
+
+def get_plugins_config_files(name, suffix=".conf"):
+ """walk through each of plugin to find all the config files in the"""
+ """name directory"""
+
+ plugins_path = setting.PLUGINS_DIR
+ files = []
+ if os.path.exists(plugins_path):
+ for plugin in os.listdir(plugins_path):
+ plugin_path = os.path.join(plugins_path, plugin)
+ plugin_config = os.path.join(plugin_path, name)
+ if os.path.exists(plugin_config):
+ for component in os.listdir(plugin_config):
+ if not component.endswith(suffix):
+ continue
+ files.append(os.path.join(plugin_config, component))
+ return files
+
+
+def load_configs(
+ config_dir, config_name_suffix='.conf',
+ env_globals={}, env_locals={}
+):
+ """Load configurations from config dir."""
+ """The config file could be in the config_dir or in plugins config_dir"""
+ """The plugins config_dir is formed as, for example /etc/compass/adapter"""
+ """Then the plugins config_dir is /etc/compass/plugins/xxx/adapter"""
+
+ # TODO(Carl) instead of using config_dir, it should use a name such as
+ # adapter etc, however, doing it requires a lot client sites changes,
+ # will do it later.
+
+ configs = []
+ config_files = []
+ config_dir = str(config_dir)
+
+ """search for config_dir"""
+ if os.path.exists(config_dir):
+ for component in os.listdir(config_dir):
+ if not component.endswith(config_name_suffix):
+ continue
+ config_files.append(os.path.join(config_dir, component))
+
+ """search for plugins config_dir"""
+ index = config_dir.rfind("/")
+
+ config_files.extend(get_plugins_config_files(config_dir[index + 1:],
+ config_name_suffix))
+
+ if not config_files:
+ logging.error('path %s and plugins does not exist', config_dir)
+ for path in config_files:
+ logging.debug('load config from %s', path)
+ config_globals = {}
+ config_globals.update(env_globals)
+ config_locals = {}
+ config_locals.update(env_locals)
+ try:
+ execfile(path, config_globals, config_locals)
+ except Exception as error:
+ logging.exception(error)
+ raise error
+ configs.append(config_locals)
+ return configs
+
+
+def pretty_print(*contents):
+ """pretty print contents."""
+ if len(contents) == 0:
+ print ""
+ else:
+ print "\n".join(content for content in contents)
+
+
+def get_switch_machines_from_file(filename):
+ """get switch machines from file."""
+ switches = []
+ switch_machines = {}
+ with open(filename) as switch_file:
+ for line in switch_file:
+ line = line.strip()
+ if not line:
+ # ignore empty line
+ continue
+
+ if line.startswith('#'):
+ # ignore comments
+ continue
+
+ columns = [column for column in line.split(',')]
+ if not columns:
+ # ignore empty line
+ continue
+
+ if columns[0] == 'switch':
+ (switch_ip, switch_vendor, switch_version,
+ switch_community, switch_state) = columns[1:]
+ switches.append({
+ 'ip': switch_ip,
+ 'vendor': switch_vendor,
+ 'credentials': {
+ 'version': switch_version,
+ 'community': switch_community,
+ },
+ 'state': switch_state,
+ })
+ elif columns[0] == 'machine':
+ switch_ip, switch_port, mac = columns[1:]
+ switch_machines.setdefault(switch_ip, []).append({
+ 'mac': mac,
+ 'port': switch_port,
+ })
+
+ return (switches, switch_machines)
+
+
+def execute_cli_by_ssh(cmd, host, username, password=None,
+ keyfile='/root/.ssh/id_rsa', nowait=False):
+ """SSH to execute script on remote machine
+
+ :param host: ip of the remote machine
+ :param username: username to access the remote machine
+ :param password: password to access the remote machine
+ :param cmd: command to execute
+
+ """
+ if not cmd:
+ logging.error("No command found!")
+ raise Exception('No command found!')
+
+ if nowait:
+ cmd = "nohup %s >/dev/null 2>&1 &" % cmd
+
+ stdin = None
+ stdout = None
+ stderr = None
+ try:
+ import paramiko
+ from paramiko import ssh_exception
+
+ client = paramiko.SSHClient()
+ client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+ if password:
+ client.connect(host, username=username, password=password)
+ else:
+ client.load_system_host_keys()
+ client.connect(
+ host, username=username,
+ key_filename=keyfile, look_for_keys=True
+ )
+ stdin, stdout, stderr = client.exec_command(cmd)
+ result = stdout.readlines()
+ logging.info("result of command '%s' is '%s'!" % (cmd, result))
+ return result
+
+ except ImportError:
+ err_msg = "Cannot find Paramiko package!"
+ logging.error(err_msg)
+ raise ImportError(err_msg)
+
+ except (ssh_exception.BadHostKeyException,
+ ssh_exception.AuthenticationException,
+ ssh_exception.SSHException):
+
+ err_msg = 'SSH connection error or command execution failed!'
+ logging.error(err_msg)
+ raise Exception(err_msg)
+
+ except Exception as exc:
+ logging.error(
+ 'Failed to execute command "%s", exception is %s' % (cmd, exc)
+ )
+ raise Exception(exc)
+
+ finally:
+ for resource in [stdin, stdout, stderr]:
+ if resource:
+ resource.close()
+
+ client.close()