diff options
Diffstat (limited to 'yardstick/common/utils.py')
| -rw-r--r-- | yardstick/common/utils.py | 349 |
1 files changed, 312 insertions, 37 deletions
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 82e20bec7..7475f6991 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -13,30 +13,35 @@ # License for the specific language governing permissions and limitations # under the License. -# yardstick comment: this is a modified copy of rally/rally/common/utils.py - -from __future__ import absolute_import -from __future__ import print_function - +import collections +from contextlib import closing import datetime import errno +import importlib +import ipaddress +import json import logging import os +import pydoc +import random +import re +import signal +import socket import subprocess import sys -import collections -import socket -import random -import ipaddress -from contextlib import closing +import time +import threading +import math import six from flask import jsonify from six.moves import configparser -from oslo_utils import importutils from oslo_serialization import jsonutils +from oslo_utils import encodeutils import yardstick +from yardstick.common import exceptions + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -69,28 +74,46 @@ def itersubclasses(cls, _seen=None): yield sub -def import_modules_from_package(package): - """Import modules from package and append into sys.modules +def import_modules_from_package(package, raise_exception=False): + """Import modules given a package name :param: package - Full package name. For example: rally.deploy.engines """ yardstick_root = os.path.dirname(os.path.dirname(yardstick.__file__)) - path = os.path.join(yardstick_root, *package.split(".")) + path = os.path.join(yardstick_root, *package.split('.')) for root, _, files in os.walk(path): - matches = (filename for filename in files if filename.endswith(".py") and - not filename.startswith("__")) - new_package = os.path.relpath(root, yardstick_root).replace(os.sep, ".") + matches = (filename for filename in files if filename.endswith('.py') + and not filename.startswith('__')) + new_package = os.path.relpath(root, yardstick_root).replace(os.sep, + '.') module_names = set( - ("{}.{}".format(new_package, filename.rsplit(".py", 1)[0]) for filename in matches)) - # find modules which haven't already been imported + '{}.{}'.format(new_package, filename.rsplit('.py', 1)[0]) + for filename in matches) + # Find modules which haven't already been imported missing_modules = module_names.difference(sys.modules) - logger.debug("importing %s", missing_modules) - # we have already checked for already imported modules, so we don't need to check again + logger.debug('Importing modules: %s', missing_modules) for module_name in missing_modules: try: - sys.modules[module_name] = importutils.import_module(module_name) - except (ImportError, SyntaxError): - logger.exception("unable to import %s", module_name) + importlib.import_module(module_name) + except (ImportError, SyntaxError) as exc: + if raise_exception: + raise exc + logger.exception('Unable to import module %s', module_name) + + +NON_NONE_DEFAULT = object() + + +def get_key_with_default(data, key, default=NON_NONE_DEFAULT): + value = data.get(key, default) + if value is NON_NONE_DEFAULT: + raise KeyError(key) + return value + + +def make_dict_from_map(data, key_map): + return {dest_key: get_key_with_default(data, src_key, default) + for dest_key, (src_key, default) in key_map.items()} def makedirs(d): @@ -109,19 +132,23 @@ def remove_file(path): raise -def execute_command(cmd): +def execute_command(cmd, **kwargs): exec_msg = "Executing command: '%s'" % cmd logger.debug(exec_msg) - output = subprocess.check_output(cmd.split()).split(os.linesep) - - return output + output = subprocess.check_output(cmd.split(), **kwargs) + return encodeutils.safe_decode(output, incoming='utf-8').split(os.linesep) def source_env(env_file): p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE, shell=True) output = p.communicate()[0] + + # sometimes output type would be binary_type, and it don't have splitlines + # method, so we need to decode + if isinstance(output, six.binary_type): + output = encodeutils.safe_decode(output) env = dict(line.split('=', 1) for line in output.splitlines() if '=' in line) os.environ.update(env) return env @@ -170,20 +197,16 @@ def parse_ini_file(path): def get_port_mac(sshclient, port): cmd = "ifconfig |grep HWaddr |grep %s |awk '{print $5}' " % port - status, stdout, stderr = sshclient.execute(cmd) + _, stdout, _ = sshclient.execute(cmd, raise_on_error=True) - if status: - raise RuntimeError(stderr) return stdout.rstrip() def get_port_ip(sshclient, port): cmd = "ifconfig %s |grep 'inet addr' |awk '{print $2}' " \ "|cut -d ':' -f2 " % port - status, stdout, stderr = sshclient.execute(cmd) + _, stdout, _ = sshclient.execute(cmd, raise_on_error=True) - if status: - raise RuntimeError(stderr) return stdout.rstrip() @@ -258,11 +281,30 @@ def get_free_port(ip): def mac_address_to_hex_list(mac): - octets = ["0x{:02x}".format(int(elem, 16)) for elem in mac.split(':')] - assert len(octets) == 6 and all(len(octet) == 4 for octet in octets) + try: + octets = ["0x{:02x}".format(int(elem, 16)) for elem in mac.split(':')] + except ValueError: + raise exceptions.InvalidMacAddress(mac_address=mac) + if len(octets) != 6 or all(len(octet) != 4 for octet in octets): + raise exceptions.InvalidMacAddress(mac_address=mac) return octets +def make_ipv4_address(ip_addr): + return ipaddress.IPv4Address(six.text_type(ip_addr)) + + +def get_ip_range_count(iprange): + start_range, end_range = iprange.split("-") + start = int(make_ipv4_address(start_range)) + end = int(make_ipv4_address(end_range)) + return end - start + + +def get_ip_range_start(iprange): + return str(make_ipv4_address(iprange.split("-")[0])) + + def safe_ip_address(ip_addr): """ get ip address version v6 or v4 """ try: @@ -283,6 +325,19 @@ def get_ip_version(ip_addr): return address.version +def make_ip_addr(ip, mask): + """ + :param ip[str]: ip adddress + :param mask[str]: /24 prefix of 255.255.255.0 netmask + :return: IPv4Interface object + """ + try: + return ipaddress.ip_interface(six.text_type('/'.join([ip, mask]))) + except (TypeError, ValueError): + # None so we can skip later + return None + + def ip_to_hex(ip_addr, separator=''): try: address = ipaddress.ip_address(six.text_type(ip_addr)) @@ -299,6 +354,14 @@ def ip_to_hex(ip_addr, separator=''): return separator.join('{:02x}'.format(octet) for octet in address.packed) +def get_mask_from_ip_range(ip_low, ip_high): + _ip_low = ipaddress.ip_address(ip_low) + _ip_high = ipaddress.ip_address(ip_high) + _ip_low_int = int(_ip_low) + _ip_high_int = int(_ip_high) + return _ip_high.max_prefixlen - (_ip_high_int ^ _ip_low_int).bit_length() + + def try_int(s, *args): """Convert to integer if possible.""" try: @@ -386,16 +449,228 @@ class ErrorClass(object): class Timer(object): - def __init__(self): + def __init__(self, timeout=None, raise_exception=True): super(Timer, self).__init__() self.start = self.delta = None + self._timeout = int(timeout) if timeout else None + self._timeout_flag = False + self._raise_exception = raise_exception + + def _timeout_handler(self, *args): + self._timeout_flag = True + if self._raise_exception: + raise exceptions.TimerTimeout(timeout=self._timeout) + self.__exit__() def __enter__(self): self.start = datetime.datetime.now() + if self._timeout: + signal.signal(signal.SIGALRM, self._timeout_handler) + signal.alarm(self._timeout) return self def __exit__(self, *_): + if self._timeout: + signal.alarm(0) self.delta = datetime.datetime.now() - self.start def __getattr__(self, item): return getattr(self.delta, item) + + def __iter__(self): + self._raise_exception = False + return self.__enter__() + + def next(self): # pragma: no cover + # NOTE(ralonsoh): Python 2 support. + if not self._timeout_flag: + return datetime.datetime.now() + raise StopIteration() + + def __next__(self): # pragma: no cover + # NOTE(ralonsoh): Python 3 support. + return self.next() + + def __del__(self): # pragma: no cover + signal.alarm(0) + + def delta_time_sec(self): + return (datetime.datetime.now() - self.start).total_seconds() + + +def read_meminfo(ssh_client): + """Read "/proc/meminfo" file and parse all keys and values""" + + cpuinfo = six.BytesIO() + ssh_client.get_file_obj('/proc/meminfo', cpuinfo) + lines = cpuinfo.getvalue().decode('utf-8') + matches = re.findall(r"([\w\(\)]+):\s+(\d+)( kB)*", lines) + output = {} + for match in matches: + output[match[0]] = match[1] + + return output + + +def setup_hugepages(ssh_client, size_kb): + """Setup needed number of hugepages for the size specified""" + + NR_HUGEPAGES_PATH = '/proc/sys/vm/nr_hugepages' + meminfo = read_meminfo(ssh_client) + hp_size_kb = int(meminfo['Hugepagesize']) + hp_number = int(math.ceil(size_kb / float(hp_size_kb))) + ssh_client.execute( + 'echo %s | sudo tee %s' % (hp_number, NR_HUGEPAGES_PATH)) + hp = six.BytesIO() + ssh_client.get_file_obj(NR_HUGEPAGES_PATH, hp) + hp_number_set = int(hp.getvalue().decode('utf-8').splitlines()[0]) + logger.info('Hugepages size (kB): %s, number claimed: %s, number set: %s', + hp_size_kb, hp_number, hp_number_set) + return hp_size_kb, hp_number, hp_number_set + + +def find_relative_file(path, task_path): + """ + Find file in one of places: in abs of path or relative to a directory path, + in this order. + + :param path: + :param task_path: + :return str: full path to file + """ + # fixme: create schema to validate all fields have been provided + for lookup in [os.path.abspath(path), os.path.join(task_path, path)]: + try: + with open(lookup): + return lookup + except IOError: + pass + raise IOError(errno.ENOENT, 'Unable to find {} file'.format(path)) + + +def open_relative_file(path, task_path): + try: + return open(path) + except IOError as e: + if e.errno == errno.ENOENT: + return open(os.path.join(task_path, path)) + raise + + +def wait_until_true(predicate, timeout=60, sleep=1, exception=None): + """Wait until callable predicate is evaluated as True + + When in a thread different from the main one, Timer(timeout) will fail + because signal is not handled. In this case + + :param predicate: (func) callable deciding whether waiting should continue + :param timeout: (int) timeout in seconds how long should function wait + :param sleep: (int) polling interval for results in seconds + :param exception: exception instance to raise on timeout. If None is passed + (default) then WaitTimeout exception is raised. + """ + if isinstance(threading.current_thread(), threading._MainThread): + try: + with Timer(timeout=timeout): + while not predicate(): + time.sleep(sleep) + except exceptions.TimerTimeout: + if exception and issubclass(exception, Exception): + raise exception # pylint: disable=raising-bad-type + raise exceptions.WaitTimeout + else: + with Timer() as timer: + while timer.delta_time_sec() < timeout: + if predicate(): + return + time.sleep(sleep) + if exception and issubclass(exception, Exception): + raise exception # pylint: disable=raising-bad-type + raise exceptions.WaitTimeout + + +def send_socket_command(host, port, command): + """Send a string command to a specific port in a host + + :param host: (str) ip or hostname of the host + :param port: (int) port number + :param command: (str) command to send + :return: 0 if success, error number if error + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ret = 0 + try: + err_number = sock.connect_ex((host, int(port))) + if err_number != 0: + return err_number + sock.sendall(six.b(command)) + except Exception: # pylint: disable=broad-except + ret = 1 + finally: + sock.close() + return ret + + +def safe_cast(value, type_to_convert, default_value): + """Convert value to type, in case of error return default_value + + :param value: value to convert + :param type_to_convert: type to convert, could be "type" or "string" + :param default_value: default value to return + :return: converted value or default_value + """ + if isinstance(type_to_convert, type): + _type = type_to_convert + else: + _type = pydoc.locate(type_to_convert) + if not _type: + raise exceptions.InvalidType(type_to_convert=type_to_convert) + + try: + return _type(value) + except ValueError: + return default_value + + +def get_os_version(ssh_client): + """Return OS version. + + :param ssh_client: SSH + :return str: Linux OS versions + """ + os_ver = ssh_client.execute("cat /etc/lsb-release")[1] + return os_ver + + +def get_kernel_version(ssh_client): + """Return kernel version. + + :param ssh_client: SSH + :return str: Linux kernel versions + """ + kernel_ver = ssh_client.execute("uname -a")[1] + return kernel_ver + + +def get_sample_vnf_info(ssh_client, + json_file='/opt/nsb_bin/yardstick_sample_vnf.json'): + """Return sample VNF data. + + :param ssh_client: SSH + :param json_file: str + :return dict: information about sample VNF + """ + rc, json_str, err = ssh_client.execute("cat %s" % json_file) + logger.debug("cat %s: %s, rc: %s, err: %s", json_file, json_str, rc, err) + + if rc: + return {} + json_data = json.loads(json_str) + for vnf_data in json_data.values(): + out = ssh_client.execute("md5sum %s" % vnf_data["path_vnf"])[1] + md5 = out.split()[0].strip() + if md5 == vnf_data["md5"]: + vnf_data["md5_result"] = "MD5 checksum is valid" + else: + vnf_data["md5_result"] = "MD5 checksum is invalid" + return json_data |