6 files changed, 826 insertions, 0 deletions
diff --git a/compass-tasks/utils/__init__.py b/compass-tasks/utils/__init__.py
new file mode 100644
index 0000000..4ee55a4
--- /dev/null
+++ b/compass-tasks/utils/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/utils/celeryconfig_wrapper.py b/compass-tasks/utils/celeryconfig_wrapper.py
new file mode 100644
index 0000000..b6644ba
--- /dev/null
+++ b/compass-tasks/utils/celeryconfig_wrapper.py
@@ -0,0 +1,44 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""celeryconfig wrapper.
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+import os.path
+import urllib
+
+from compass.utils import setting_wrapper as setting
+
+
+# CELERY_RESULT_BACKEND = 'amqp://'
+
+# BROKER_URL = 'amqp://guest:guest@localhost:5672//'
+
+
+CELERY_IMPORTS = ('compass.tasks.tasks',)
+
+
+if setting.CELERYCONFIG_FILE:
+    CELERY_CONFIG = os.path.join(
+        str(setting.CELERYCONFIG_DIR),
+        str(setting.CELERYCONFIG_FILE))
+
+    try:
+        logging.info('load celery config from %s', CELERY_CONFIG)
+        execfile(CELERY_CONFIG, globals(), locals())
+    except Exception as error:
+        logging.exception(error)
+        raise error
diff --git a/compass-tasks/utils/flags.py b/compass-tasks/utils/flags.py
new file mode 100644
index 0000000..a3169f5
--- /dev/null
+++ b/compass-tasks/utils/flags.py
@@ -0,0 +1,91 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to load flags.
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import sys
+
+from optparse import OptionParser
+
+
+class Flags(object):
+    """Class to store flags."""
+
+    PARSER = OptionParser()
+    PARSED_OPTIONS = None
+
+    @classmethod
+    def parse_args(cls):
+        """parse args."""
+        (options, argv) = Flags.PARSER.parse_args()
+        sys.argv = [sys.argv[0]] + argv
+        Flags.PARSED_OPTIONS = options
+
+    def __getattr__(self, name):
+        if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name):
+            return getattr(Flags.PARSED_OPTIONS, name)
+
+        for option in Flags.PARSER.option_list:
+            if option.dest == name:
+                return option.default
+
+        raise AttributeError('Option instance has no attribute %s' % name)
+
+    def __setattr__(self, name, value):
+        if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name):
+            setattr(Flags.PARSED_OPTIONS, name, value)
+            return
+
+        for option in Flags.PARSER.option_list:
+            if option.dest == name:
+                option.default = value
+                return
+
+        object.__setattr__(self, name, value)
+
+
+OPTIONS = Flags()
+
+
+def init():
+    """Init flag parsing."""
+    OPTIONS.parse_args()
+
+
+def add(flagname, **kwargs):
+    """Add a flag name and its setting.
+
+    :param flagname: flag name declared in cmd as --<flagname>=...
+    :type flagname: str
+    """
+    Flags.PARSER.add_option('--%s' % flagname,
+                            dest=flagname, **kwargs)
+
+
+def add_bool(flagname, default=True, **kwargs):
+    """Add a bool flag name and its setting.
+
+    :param flagname: flag name declared in cmd as --[no]<flagname>.
+    :type flagname: str
+    :param default: default value
+    :type default: bool
+    """
+    Flags.PARSER.add_option('--%s' % flagname,
+                            dest=flagname, default=default,
+                            action="store_true", **kwargs)
+    Flags.PARSER.add_option('--no%s' % flagname,
+                            dest=flagname,
+                            action="store_false", **kwargs)
diff --git a/compass-tasks/utils/logsetting.py b/compass-tasks/utils/logsetting.py
new file mode 100644
index 0000000..836ebcb
--- /dev/null
+++ b/compass-tasks/utils/logsetting.py
@@ -0,0 +1,108 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to setup logging configuration.
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+
+import logging
+import logging.handlers
+import os
+import os.path
+import sys
+
+from compass.utils import flags
+from compass.utils import setting_wrapper as setting
+
+
+flags.add('loglevel',
+          help='logging level', default=setting.DEFAULT_LOGLEVEL)
+flags.add('logdir',
+          help='logging directory', default=setting.DEFAULT_LOGDIR)
+flags.add('logfile',
+          help='logging filename', default=None)
+flags.add('log_interval', type='int',
+          help='log interval', default=setting.DEFAULT_LOGINTERVAL)
+flags.add('log_interval_unit',
+          help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT)
+flags.add('log_format',
+          help='log format', default=setting.DEFAULT_LOGFORMAT)
+flags.add('log_backup_count', type='int',
+          help='log backup count', default=setting.DEFAULT_LOGBACKUPCOUNT)
+
+
+# mapping str setting in flag --loglevel to logging level.
+LOGLEVEL_MAPPING = {
+    'finest': logging.DEBUG - 2,  # more detailed log.
+    'fine': logging.DEBUG - 1,    # detailed log.
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+    'critical': logging.CRITICAL,
+}
+
+
+logging.addLevelName(LOGLEVEL_MAPPING['fine'], 'fine')
+logging.addLevelName(LOGLEVEL_MAPPING['finest'], 'finest')
+
+
+# disable logging when logsetting.init not called
+logging.getLogger().setLevel(logging.CRITICAL)
+
+
+def getLevelByName(level_name):
+    """Get log level by level name."""
+    return LOGLEVEL_MAPPING[level_name]
+
+
+def init():
+    """Init loggsetting. It should be called after flags.init."""
+    loglevel = flags.OPTIONS.loglevel.lower()
+    logdir = flags.OPTIONS.logdir
+    logfile = flags.OPTIONS.logfile
+    logger = logging.getLogger()
+    if logger.handlers:
+        for handler in logger.handlers:
+            logger.removeHandler(handler)
+
+    if logdir:
+        if not logfile:
+            logfile = '%s.log' % os.path.basename(sys.argv[0])
+
+        handler = logging.handlers.TimedRotatingFileHandler(
+            os.path.join(logdir, logfile),
+            when=flags.OPTIONS.log_interval_unit,
+            interval=flags.OPTIONS.log_interval,
+            backupCount=flags.OPTIONS.log_backup_count)
+    else:
+        if not logfile:
+            handler = logging.StreamHandler(sys.stderr)
+        else:
+            handler = logging.handlers.TimedRotatingFileHandler(
+                logfile,
+                when=flags.OPTIONS.log_interval_unit,
+                interval=flags.OPTIONS.log_interval,
+                backupCount=flags.OPTIONS.log_backup_count)
+
+    if loglevel in LOGLEVEL_MAPPING:
+        logger.setLevel(LOGLEVEL_MAPPING[loglevel])
+        handler.setLevel(LOGLEVEL_MAPPING[loglevel])
+
+    formatter = logging.Formatter(
+        flags.OPTIONS.log_format)
+
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
diff --git a/compass-tasks/utils/setting_wrapper.py b/compass-tasks/utils/setting_wrapper.py
new file mode 100644
index 0000000..0b3e9f7
--- /dev/null
+++ b/compass-tasks/utils/setting_wrapper.py
@@ -0,0 +1,175 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""comapss setting wrapper.
+
+   .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com>
+"""
+import datetime
+import lazypy
+import logging
+import os
+import os.path
+
+
+# default setting
+CONFIG_DIR = os.environ.get('COMPASS_CONFIG_DIR', '/etc/compass')
+SQLALCHEMY_DATABASE_URI = 'sqlite://'
+SQLALCHEMY_DATABASE_POOL_TYPE = 'static'
+COBBLER_INSTALLATION_LOGDIR = '/var/log/cobbler/anamon'
+CHEF_INSTALLATION_LOGDIR = '/var/log/chef'
+INSTALLATION_LOGDIR = {
+    'CobblerInstaller': COBBLER_INSTALLATION_LOGDIR,
+    'ChefInstaller': CHEF_INSTALLATION_LOGDIR
+}
+CLUSTERHOST_INATALLATION_LOGDIR_NAME = 'name'
+HOST_INSTALLATION_LOGDIR_NAME = 'name'
+DEFAULT_LOGLEVEL = 'debug'
+DEFAULT_LOGDIR = '/tmp'
+DEFAULT_LOGINTERVAL = 1
+DEFAULT_LOGINTERVAL_UNIT = 'h'
+DEFAULT_LOGFORMAT = (
+    '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s')
+DEFAULT_LOGBACKUPCOUNT = 5
+WEB_LOGFILE = ''
+CELERY_LOGFILE = ''
+CELERYCONFIG_DIR = lazypy.delay(lambda: CONFIG_DIR)
+CELERYCONFIG_FILE = ''
+PROGRESS_UPDATE_INTERVAL = 30
+POLLSWITCH_INTERVAL = 60
+SWITCHES = [
+]
+
+USER_AUTH_HEADER_NAME = 'X-Auth-Token'
+USER_TOKEN_DURATION = '2h'
+COMPASS_ADMIN_EMAIL = 'admin@huawei.com'
+COMPASS_ADMIN_PASSWORD = 'admin'
+COMPASS_DEFAULT_PERMISSIONS = [
+    'list_permissions',
+]
+SWITCHES_DEFAULT_FILTERS = []
+DEFAULT_SWITCH_IP = '0.0.0.0'
+DEFAULT_SWITCH_PORT = 0
+
+COMPASS_SUPPORTED_PROXY = 'http://127.0.0.1:3128'
+COMPASS_SUPPORTED_DEFAULT_NOPROXY = ['127.0.0.1']
+COMPASS_SUPPORTED_NTP_SERVER = '127.0.0.1'
+COMPASS_SUPPORTED_DNS_SERVERS = ['127.0.0.1']
+COMPASS_SUPPORTED_DOMAINS = []
+COMPASS_SUPPORTED_DEFAULT_GATEWAY = '127.0.0.1'
+COMPASS_SUPPORTED_LOCAL_REPO = 'http://127.0.0.1'
+
+PROGRESS_UPDATE_PID_FILE = '/var/run/progress_update.pid'
+
+PROXY_URL_PREFIX = 'http://10.145.81.205:5000'
+
+OS_INSTALLER_DIR = ''
+PACKAGE_INSTALLER_DIR = ''
+OS_DIR = ''
+ADAPTER_DIR = ''
+OS_METADATA_DIR = ''
+PACKAGE_METADATA_DIR = ''
+FLAVOR_METADATA_DIR = ''
+OS_FIELD_DIR = ''
+PACKAGE_FIELD_DIR = ''
+FLAVOR_FIELD_DIR = ''
+ADAPTER_ROLE_DIR = ''
+ADAPTER_FLAVOR_DIR = ''
+VALIDATOR_DIR = ''
+CALLBACK_DIR = ''
+TMPL_DIR = ''
+MACHINE_LIST_DIR = ''
+PROGRESS_CALCULATOR_DIR = ''
+OS_MAPPING_DIR = ''
+FLAVOR_MAPPING_DIR = ''
+PLUGINS_DIR = ''
+
+if (
+    'COMPASS_IGNORE_SETTING' in os.environ and
+    os.environ['COMPASS_IGNORE_SETTING']
+):
+    pass
+else:
+    if 'COMPASS_SETTING' in os.environ:
+        SETTING = os.environ['COMPASS_SETTING']
+    else:
+        SETTING = '/etc/compass/setting'
+
+    try:
+        logging.info('load setting from %s', SETTING)
+        execfile(SETTING, globals(), locals())
+    except Exception as error:
+        logging.exception(error)
+        raise error
+
+if not OS_INSTALLER_DIR:
+    OS_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'os_installer')
+
+if not PACKAGE_INSTALLER_DIR:
+    PACKAGE_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'package_installer')
+
+if not OS_DIR:
+    OS_DIR = os.path.join(CONFIG_DIR, 'os')
+
+if not ADAPTER_DIR:
+    ADAPTER_DIR = os.path.join(CONFIG_DIR, 'adapter')
+
+if not OS_METADATA_DIR:
+    OS_METADATA_DIR = os.path.join(CONFIG_DIR, 'os_metadata')
+
+if not PACKAGE_METADATA_DIR:
+    PACKAGE_METADATA_DIR = os.path.join(CONFIG_DIR, 'package_metadata')
+
+if not FLAVOR_METADATA_DIR:
+    FLAVOR_METADATA_DIR = os.path.join(CONFIG_DIR, 'flavor_metadata')
+
+if not OS_FIELD_DIR:
+    OS_FIELD_DIR = os.path.join(CONFIG_DIR, 'os_field')
+
+if not PACKAGE_FIELD_DIR:
+    PACKAGE_FIELD_DIR = os.path.join(CONFIG_DIR, 'package_field')
+
+if not FLAVOR_FIELD_DIR:
+    FLAVOR_FIELD_DIR = os.path.join(CONFIG_DIR, 'flavor_field')
+
+if not ADAPTER_ROLE_DIR:
+    ADAPTER_ROLE_DIR = os.path.join(CONFIG_DIR, 'role')
+
+if not ADAPTER_FLAVOR_DIR:
+    ADAPTER_FLAVOR_DIR = os.path.join(CONFIG_DIR, 'flavor')
+
+if not VALIDATOR_DIR:
+    VALIDATOR_DIR = os.path.join(CONFIG_DIR, 'validator')
+
+if not CALLBACK_DIR:
+    CALLBACK_DIR = os.path.join(CONFIG_DIR, 'callback')
+
+if not TMPL_DIR:
+    TMPL_DIR = os.path.join(CONFIG_DIR, 'templates')
+
+if not MACHINE_LIST_DIR:
+    MACHINE_LIST_DIR = os.path.join(CONFIG_DIR, 'machine_list')
+
+if not PROGRESS_CALCULATOR_DIR:
+    PROGRESS_CALCULATOR_DIR = os.path.join(CONFIG_DIR, 'progress_calculator')
+
+if not OS_MAPPING_DIR:
+    OS_MAPPING_DIR = os.path.join(CONFIG_DIR, 'os_mapping')
+
+if not FLAVOR_MAPPING_DIR:
+    FLAVOR_MAPPING_DIR = os.path.join(CONFIG_DIR, 'flavor_mapping')
+
+if not PLUGINS_DIR:
+    PLUGINS_DIR = os.environ.get('COMPASS_PLUGINS_DIR',
+                                 os.path.join(CONFIG_DIR, 'plugins'))
diff --git a/compass-tasks/utils/util.py b/compass-tasks/utils/util.py
new file mode 100644
index 0000000..39978ca
--- /dev/null
+++ b/compass-tasks/utils/util.py
@@ -0,0 +1,395 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to provider util functions in all compass code
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+
+import crypt
+import datetime
+import logging
+import os
+import os.path
+import re
+import setting_wrapper as setting
+import sys
+import warnings
+
+
+def deprecated(func):
+    """This is a decorator which can be used to mark functions as deprecated.
+
+    It will result in a warning being emitted when the function is used.
+    """
+    def new_func(*args, **kwargs):
+        warnings.warn(
+            "Call to deprecated function %s." % func.__name__,
+            category=DeprecationWarning
+        )
+        return func(*args, **kwargs)
+
+    new_func.__name__ = func.__name__
+    new_func.__doc__ = func.__doc__
+    new_func.__dict__.update(func.__dict__)
+    return new_func
+
+
+def parse_datetime(date_time, exception_class=Exception):
+    """Parse datetime str to get datetime object.
+
+    The date time format is %Y-%m-%d %H:%M:%S
+    """
+    try:
+        return datetime.datetime.strptime(
+            date_time, '%Y-%m-%d %H:%M:%S'
+        )
+    except Exception as error:
+        logging.exception(error)
+        raise exception_class(
+            'date time %s format is invalid' % date_time
+        )
+
+
+def parse_datetime_range(date_time_range, exception_class=Exception):
+    """parse datetime range str to pair of datetime objects.
+
+    The date time range format is %Y-%m-%d %H:%M:%S,%Y-%m-%d %H:%M:%S
+    """
+    try:
+        start, end = date_time_range.split(',')
+    except Exception as error:
+        logging.exception(error)
+        raise exception_class(
+            'there is no `,` in date time range %s' % date_time_range
+        )
+    if start:
+        start_datetime = parse_datetime(start, exception_class)
+    else:
+        start_datetime = None
+    if end:
+        end_datetime = parse_datetime(end, exception_class)
+    else:
+        end_datetime = None
+    return start_datetime, end_datetime
+
+
+def parse_request_arg_dict(arg, exception_class=Exception):
+    """parse string to dict.
+
+    The str is formatted like a=b;c=d and parsed to
+    {'a': 'b', 'c': 'd'}
+    """
+    arg_dict = {}
+    arg_pairs = arg.split(';')
+    for arg_pair in arg_pairs:
+        try:
+            arg_name, arg_value = arg_pair.split('=', 1)
+        except Exception as error:
+            logging.exception(error)
+            raise exception_class(
+                'there is no `=` in %s' % arg_pair
+            )
+        arg_dict[arg_name] = arg_value
+    return arg_dict
+
+
+def format_datetime(date_time):
+    """Generate string from datetime object."""
+    return date_time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def merge_dict(lhs, rhs, override=True):
+    """Merge nested right dict into left nested dict recursively.
+
+    :param lhs: dict to be merged into.
+    :type lhs: dict
+    :param rhs: dict to merge from.
+    :type rhs: dict
+    :param override: the value in rhs overide the value in left if True.
+    :type override: boolean
+    """
+    if not isinstance(lhs, dict) or not isinstance(rhs, dict):
+        if override:
+            return rhs
+        else:
+            return lhs
+
+    for key, value in rhs.items():
+        if key not in lhs:
+            lhs[key] = rhs[key]
+        else:
+            lhs[key] = merge_dict(lhs[key], value, override)
+
+    return lhs
+
+
+def recursive_merge_dict(name, all_dicts, parents):
+    """Recursively merge parent dict into base dict."""
+    parent_name = parents.get(name, None)
+    base_dict = all_dicts.get(name, {})
+    if not parent_name:
+        return base_dict
+    merged = recursive_merge_dict(parent_name, all_dicts, parents)
+    return merge_dict(base_dict, merged, override=False)
+
+
+def encrypt(value, crypt_method=None):
+    """Get encrypted value."""
+    if not crypt_method:
+        if hasattr(crypt, 'METHOD_MD5'):
+            crypt_method = crypt.METHOD_MD5
+        else:
+            # for python2.7, copy python2.6 METHOD_MD5 logic here.
+            from random import choice
+            import string
+
+            _saltchars = string.ascii_letters + string.digits + './'
+
+            def _mksalt():
+                """generate salt."""
+                salt = '$1$'
+                salt += ''.join(choice(_saltchars) for _ in range(8))
+                return salt
+
+            crypt_method = _mksalt()
+
+    return crypt.crypt(value, crypt_method)
+
+
+def parse_time_interval(time_interval_str):
+    """parse string of time interval to time interval.
+
+    supported time interval unit: ['d', 'w', 'h', 'm', 's']
+    Examples:
+       time_interval_str: '3d 2h' time interval to 3 days and 2 hours.
+    """
+    if not time_interval_str:
+        return 0
+
+    time_interval_tuple = [
+        time_interval_element
+        for time_interval_element in time_interval_str.split(' ')
+        if time_interval_element
+    ]
+    time_interval_dict = {}
+    time_interval_unit_mapping = {
+        'd': 'days',
+        'w': 'weeks',
+        'h': 'hours',
+        'm': 'minutes',
+        's': 'seconds'
+    }
+    for time_interval_element in time_interval_tuple:
+        mat = re.match(r'^([+-]?\d+)(w|d|h|m|s).*', time_interval_element)
+        if not mat:
+            continue
+
+        time_interval_value = int(mat.group(1))
+        time_interval_unit = time_interval_unit_mapping[mat.group(2)]
+        time_interval_dict[time_interval_unit] = (
+            time_interval_dict.get(time_interval_unit, 0) + time_interval_value
+        )
+
+    time_interval = datetime.timedelta(**time_interval_dict)
+    if sys.version_info[0:2] > (2, 6):
+        return time_interval.total_seconds()
+    else:
+        return (
+            time_interval.microseconds + (
+                time_interval.seconds + time_interval.days * 24 * 3600
+            ) * 1e6
+        ) / 1e6
+
+
+def get_plugins_config_files(name, suffix=".conf"):
+    """walk through each of plugin to find all the config files in the"""
+    """name directory"""
+
+    plugins_path = setting.PLUGINS_DIR
+    files = []
+    if os.path.exists(plugins_path):
+        for plugin in os.listdir(plugins_path):
+            plugin_path = os.path.join(plugins_path, plugin)
+            plugin_config = os.path.join(plugin_path, name)
+            if os.path.exists(plugin_config):
+                for component in os.listdir(plugin_config):
+                    if not component.endswith(suffix):
+                        continue
+                    files.append(os.path.join(plugin_config, component))
+    return files
+
+
+def load_configs(
+    config_dir, config_name_suffix='.conf',
+    env_globals={}, env_locals={}
+):
+    """Load configurations from config dir."""
+    """The config file could be in the config_dir or in plugins config_dir"""
+    """The plugins config_dir is formed as, for example /etc/compass/adapter"""
+    """Then the plugins config_dir is /etc/compass/plugins/xxx/adapter"""
+
+    # TODO(Carl) instead of using config_dir, it should use a name such as
+    # adapter etc, however, doing it requires a lot client sites changes,
+    # will do it later.
+
+    configs = []
+    config_files = []
+    config_dir = str(config_dir)
+
+    """search for config_dir"""
+    if os.path.exists(config_dir):
+        for component in os.listdir(config_dir):
+            if not component.endswith(config_name_suffix):
+                continue
+            config_files.append(os.path.join(config_dir, component))
+
+    """search for plugins config_dir"""
+    index = config_dir.rfind("/")
+
+    config_files.extend(get_plugins_config_files(config_dir[index + 1:],
+                                                 config_name_suffix))
+
+    if not config_files:
+        logging.error('path %s and plugins does not exist', config_dir)
+    for path in config_files:
+        logging.debug('load config from %s', path)
+        config_globals = {}
+        config_globals.update(env_globals)
+        config_locals = {}
+        config_locals.update(env_locals)
+        try:
+            execfile(path, config_globals, config_locals)
+        except Exception as error:
+            logging.exception(error)
+            raise error
+        configs.append(config_locals)
+    return configs
+
+
+def pretty_print(*contents):
+    """pretty print contents."""
+    if len(contents) == 0:
+        print ""
+    else:
+        print "\n".join(content for content in contents)
+
+
+def get_switch_machines_from_file(filename):
+    """get switch machines from file."""
+    switches = []
+    switch_machines = {}
+    with open(filename) as switch_file:
+        for line in switch_file:
+            line = line.strip()
+            if not line:
+                # ignore empty line
+                continue
+
+            if line.startswith('#'):
+                # ignore comments
+                continue
+
+            columns = [column for column in line.split(',')]
+            if not columns:
+                # ignore empty line
+                continue
+
+            if columns[0] == 'switch':
+                (switch_ip, switch_vendor, switch_version,
+                 switch_community, switch_state) = columns[1:]
+                switches.append({
+                    'ip': switch_ip,
+                    'vendor': switch_vendor,
+                    'credentials': {
+                        'version': switch_version,
+                        'community': switch_community,
+                    },
+                    'state': switch_state,
+                })
+            elif columns[0] == 'machine':
+                switch_ip, switch_port, mac = columns[1:]
+                switch_machines.setdefault(switch_ip, []).append({
+                    'mac': mac,
+                    'port': switch_port,
+                })
+
+    return (switches, switch_machines)
+
+
+def execute_cli_by_ssh(cmd, host, username, password=None,
+                       keyfile='/root/.ssh/id_rsa', nowait=False):
+    """SSH to execute script on remote machine
+
+    :param host: ip of the remote machine
+    :param username: username to access the remote machine
+    :param password: password to access the remote machine
+    :param cmd: command to execute
+
+    """
+    if not cmd:
+        logging.error("No command found!")
+        raise Exception('No command found!')
+
+    if nowait:
+        cmd = "nohup %s >/dev/null 2>&1 &" % cmd
+
+    stdin = None
+    stdout = None
+    stderr = None
+    try:
+        import paramiko
+        from paramiko import ssh_exception
+
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        if password:
+            client.connect(host, username=username, password=password)
+        else:
+            client.load_system_host_keys()
+            client.connect(
+                host, username=username,
+                key_filename=keyfile, look_for_keys=True
+            )
+        stdin, stdout, stderr = client.exec_command(cmd)
+        result = stdout.readlines()
+        logging.info("result of command '%s' is '%s'!" % (cmd, result))
+        return result
+
+    except ImportError:
+        err_msg = "Cannot find Paramiko package!"
+        logging.error(err_msg)
+        raise ImportError(err_msg)
+
+    except (ssh_exception.BadHostKeyException,
+            ssh_exception.AuthenticationException,
+            ssh_exception.SSHException):
+
+        err_msg = 'SSH connection error or command execution failed!'
+        logging.error(err_msg)
+        raise Exception(err_msg)
+
+    except Exception as exc:
+        logging.error(
+            'Failed to execute command "%s", exception is %s' % (cmd, exc)
+        )
+        raise Exception(exc)
+
+    finally:
+        for resource in [stdin, stdout, stderr]:
+            if resource:
+                resource.close()
+
+        client.close()