diff options
Diffstat (limited to 'cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util')
6 files changed, 528 insertions, 0 deletions
diff --git a/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/__init__.py b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/__init__.py diff --git a/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/common_fun.py b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/common_fun.py new file mode 100644 index 0000000..34f3e87 --- /dev/null +++ b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/common_fun.py @@ -0,0 +1,206 @@ +"""
+Utils for SPDK driver.
+"""
+
+import glob
+import os
+import re
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from cyborg.accelerator import configuration
+from cyborg.accelerator.common import exception
+from cyborg.accelerator.drivers.spdk.util.pyspdk.py_spdk import PySPDK
+from cyborg.common.i18n import _
+from pyspdk.nvmf_client import NvmfTgt
+from pyspdk.vhost_client import VhostTgt
+
+LOG = logging.getLogger(__name__)
+
+accelerator_opts = [
+ cfg.StrOpt('spdk_conf_file',
+ default='/etc/cyborg/spdk.conf',
+ help=_('SPDK conf file to be used for the SPDK driver')),
+
+ cfg.StrOpt('accelerator_servers',
+ default=['vhost', 'nvmf', 'iscsi'],
+ help=_('A list of accelerator servers to enable by default')),
+
+ cfg.StrOpt('spdk_dir',
+ default='/home/wewe/spdk',
+ help=_('The SPDK directory is /home/{user_name}/spdk')),
+
+ cfg.StrOpt('device_type',
+ default='NVMe',
+ help=_('Backend device type is NVMe by default')),
+
+ cfg.BoolOpt('remoteable',
+ default=False,
+ help=_('Remoteable is false by default'))
+]
+
+CONF = cfg.CONF
+CONF.register_opts(accelerator_opts, group=configuration.SHARED_CONF_GROUP)
+
+config = configuration.Configuration(accelerator_opts)
+config.append_config_values(accelerator_opts)
+SERVERS = config.safe_get('accelerator_servers')
+SERVERS_PATTERN = re.compile("|".join(["(%s)" % s for s in SERVERS]))
+SPDK_SERVER_APP_DIR = os.path.join(config.safe_get('spdk_dir'), 'app/')
+
+
+def discover_servers():
+ """Discover backend servers according to the CONF
+ :returns: server list.
+ """
+ servers = set()
+ for p in glob.glob1(SPDK_SERVER_APP_DIR, "*"):
+ m = SERVERS_PATTERN.match(p)
+ if m:
+ servers.add(m.group())
+ return list(servers)
+
+
+def delete_bdev(py, accelerator, name):
+ """Delete a blockdev
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :param name: Blockdev name to be deleted.
+ """
+ acc_client = get_accelerator_client(py, accelerator)
+ acc_client.delete_bdev(name)
+
+
+def kill_instance(py, accelerator, sig_name):
+ """Send signal to instance
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :param sig_name: signal will be sent to server.
+ """
+ acc_client = get_accelerator_client(py, accelerator)
+ acc_client.kill_instance(sig_name)
+
+
+def construct_aio_bdev(py, accelerator, filename, name, block_size):
+ """Add a bdev with aio backend
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :param filename: Path to device or file (ex: /dev/sda).
+ :param name: Block device name.
+ :param block_size: Block size for this bdev.
+ :return: name.
+ """
+ acc_client = get_accelerator_client(py, accelerator)
+ acc_client.construct_aio_bdev(filename, name, block_size)
+ return name
+
+
+def construct_error_bdev(py, accelerator, basename):
+ """Add a bdev with error backend
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :param basename: Path to device or file (ex: /dev/sda).
+ """
+ acc_client = get_accelerator_client(py, accelerator)
+ acc_client.construct_error_bdev(basename)
+
+
+def construct_nvme_bdev(py,
+ accelerator,
+ name,
+ trtype,
+ traddr,
+ adrfam,
+ trsvcid,
+ subnqn
+ ):
+ """Add a bdev with nvme backend
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :param name: Name of the bdev.
+ :param trtype: NVMe-oF target trtype: e.g., rdma, pcie.
+ :param traddr: NVMe-oF target address: e.g., an ip address
+ or BDF.
+ :param adrfam: NVMe-oF target adrfam: e.g., ipv4, ipv6, ib,
+ fc, intra_host.
+ :param trsvcid: NVMe-oF target trsvcid: e.g., a port number.
+ :param subnqn: NVMe-oF target subnqn.
+ :return: name.
+ """
+ acc_client = get_accelerator_client(py, accelerator)
+ acc_client.construct_nvme_bdev(name,
+ trtype,
+ traddr,
+ adrfam,
+ trsvcid,
+ subnqn
+ )
+ return name
+
+
+def construct_null_bdev(py,
+ accelerator,
+ name,
+ total_size,
+ block_size
+ ):
+ """Add a bdev with null backend
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :param name: Block device name.
+ :param total_size: Size of null bdev in MB (int > 0).
+ :param block_size: Block size for this bdev.
+ :return: name.
+ """
+ acc_client = get_accelerator_client(py, accelerator)
+ acc_client.construct_null_bdev(name, total_size, block_size)
+ return name
+
+
+def get_py_client(server):
+ """Get the py_client instance
+ :param server: server.
+ :return: Boolean.
+ :raise: InvalidAccelerator.
+ """
+ if server in SERVERS:
+ py = PySPDK(server)
+ return py
+ else:
+ msg = (_("Could not find %s accelerator") % server)
+ raise exception.InvalidAccelerator(msg)
+
+
+def check_for_setup_error(py, server):
+ """Check server's status
+ :param py: py_client.
+ :param server: server.
+ :return: Boolean.
+ :raise: AcceleratorException.
+ """
+ if py.is_alive():
+ return True
+ else:
+ msg = (_("%s accelerator is down") % server)
+ raise exception.AcceleratorException(msg)
+
+
+def get_accelerator_client(py, accelerator):
+ """Get the specific client that communicates with server
+ :param py: py_client.
+ :param accelerator: accelerator.
+ :return: acc_client.
+ :raise: InvalidAccelerator.
+ """
+ acc_client = None
+ if accelerator == 'vhost':
+ acc_client = VhostTgt(py)
+ return acc_client
+ elif accelerator == 'nvmf':
+ acc_client = NvmfTgt(py)
+ return acc_client
+ else:
+ exc_msg = (_("accelerator_client %(acc_client) is missing")
+ % acc_client)
+ raise exception.InvalidAccelerator(exc_msg)
\ No newline at end of file diff --git a/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/__init__.py b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/__init__.py diff --git a/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/nvmf_client.py b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/nvmf_client.py new file mode 100644 index 0000000..840087f --- /dev/null +++ b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/nvmf_client.py @@ -0,0 +1,119 @@ +import json
+
+
+class NvmfTgt(object):
+
+ def __init__(self, py):
+ super(NvmfTgt, self).__init__()
+ self.py = py
+
+ def get_rpc_methods(self):
+ rpc_methods = self._get_json_objs(
+ 'get_rpc_methods', '10.0.2.15')
+ return rpc_methods
+
+ def get_bdevs(self):
+ block_devices = self._get_json_objs(
+ 'get_bdevs', '10.0.2.15')
+ return block_devices
+
+ def delete_bdev(self, name):
+ sub_args = [name]
+ res = self.py.exec_rpc('delete_bdev', '10.0.2.15', sub_args=sub_args)
+ print res
+
+ def kill_instance(self, sig_name):
+ sub_args = [sig_name]
+ res = self.py.exec_rpc('kill_instance', '10.0.2.15', sub_args=sub_args)
+ print res
+
+ def construct_aio_bdev(self, filename, name, block_size):
+ sub_args = [filename, name, str(block_size)]
+ res = self.py.exec_rpc(
+ 'construct_aio_bdev',
+ '10.0.2.15',
+ sub_args=sub_args)
+ print res
+
+ def construct_error_bdev(self, basename):
+ sub_args = [basename]
+ res = self.py.exec_rpc(
+ 'construct_error_bdev',
+ '10.0.2.15',
+ sub_args=sub_args)
+ print res
+
+ def construct_nvme_bdev(
+ self,
+ name,
+ trtype,
+ traddr,
+ adrfam=None,
+ trsvcid=None,
+ subnqn=None):
+ sub_args = ["-b", "-t", "-a"]
+ sub_args.insert(1, name)
+ sub_args.insert(2, trtype)
+ sub_args.insert(3, traddr)
+ if adrfam is not None:
+ sub_args.append("-f")
+ sub_args.append(adrfam)
+ if trsvcid is not None:
+ sub_args.append("-s")
+ sub_args.append(trsvcid)
+ if subnqn is not None:
+ sub_args.append("-n")
+ sub_args.append(subnqn)
+ res = self.py.exec_rpc(
+ 'construct_nvme_bdev',
+ '10.0.2.15',
+ sub_args=sub_args)
+ return res
+
+ def construct_null_bdev(self, name, total_size, block_size):
+ sub_args = [name, str(total_size), str(block_size)]
+ res = self.py.exec_rpc(
+ 'construct_null_bdev',
+ '10.0.2.15',
+ sub_args=sub_args)
+ return res
+
+ def construct_malloc_bdev(self, total_size, block_size):
+ sub_args = [str(total_size), str(block_size)]
+ res = self.py.exec_rpc(
+ 'construct_malloc_bdev',
+ '10.0.2.15',
+ sub_args=sub_args)
+ print res
+
+ def delete_nvmf_subsystem(self, nqn):
+ sub_args = [nqn]
+ res = self.py.exec_rpc(
+ 'delete_nvmf_subsystem',
+ '10.0.2.15',
+ sub_args=sub_args)
+ print res
+
+ def construct_nvmf_subsystem(
+ self,
+ nqn,
+ listen,
+ hosts,
+ serial_number,
+ namespaces):
+ sub_args = [nqn, listen, hosts, serial_number, namespaces]
+ res = self.py.exec_rpc(
+ 'construct_nvmf_subsystem',
+ '10.0.2.15',
+ sub_args=sub_args)
+ print res
+
+ def get_nvmf_subsystems(self):
+ subsystems = self._get_json_objs(
+ 'get_nvmf_subsystems', '10.0.2.15')
+ return subsystems
+
+ def _get_json_objs(self, method, server_ip):
+ res = self.py.exec_rpc(method, server_ip)
+ json_obj = json.loads(res)
+ return json_obj
\ No newline at end of file diff --git a/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/py_spdk.py b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/py_spdk.py new file mode 100644 index 0000000..a298c18 --- /dev/null +++ b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/py_spdk.py @@ -0,0 +1,82 @@ +import psutil
+import re
+import os
+import subprocess
+
+
+class PySPDK(object):
+
+ def __init__(self, pname):
+ super(PySPDK, self).__init__()
+ self.pid = None
+ self.pname = pname
+
+ def start_server(self, spdk_dir, server_name):
+ if not self.is_alive():
+ self.init_hugepages(spdk_dir)
+ server_dir = os.path.join(spdk_dir, 'app/')
+ file_dir = self._search_file(server_dir, server_name)
+ print file_dir
+ os.chdir(file_dir)
+ p = subprocess.Popen(
+ 'sudo ./%s' % server_name,
+ shell=True, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = p.communicate()
+ return out
+
+ def init_hugepages(self, spdk_dir):
+ huge_dir = os.path.join(spdk_dir, 'scripts/')
+ file_dir = self._search_file(huge_dir, 'setup.sh')
+ print file_dir
+ os.chdir(file_dir)
+ p = subprocess.Popen(
+ 'sudo ./setup.sh',
+ shell=True, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = p.communicate()
+ return out
+
+ @staticmethod
+ def _search_file(spdk_dir, file_name):
+ for dirpath, dirnames, filenames in os.walk(spdk_dir):
+ for filename in filenames:
+ if filename == file_name:
+ return dirpath
+
+ def _get_process_id(self):
+ for proc in psutil.process_iter():
+ try:
+ pinfo = proc.as_dict(attrs=['pid', 'cmdline'])
+ if re.search(self.pname, str(pinfo.get('cmdline'))):
+ self.pid = pinfo.get('pid')
+ return self.pid
+ except psutil.NoSuchProcess:
+ print "NoSuchProcess:%s" % self.pname
+ print "NoSuchProcess:%s" % self.pname
+ return self.pid
+
+ def is_alive(self):
+ self.pid = self._get_process_id()
+ if self.pid:
+ p = psutil.Process(self.pid)
+ if p.is_running():
+ return True
+ return False
+
+ @staticmethod
+ def exec_rpc(method, server='127.0.0.1', port=5260, sub_args=None):
+ exec_cmd = ["./rpc.py", "-s", "-p"]
+ exec_cmd.insert(2, server)
+ exec_cmd.insert(4, str(port))
+ exec_cmd.insert(5, method)
+ if sub_args is None:
+ sub_args = []
+ exec_cmd.extend(sub_args)
+ p = subprocess.Popen(
+ exec_cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ out, err = p.communicate()
+ return out
\ No newline at end of file diff --git a/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/vhost_client.py b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/vhost_client.py new file mode 100644 index 0000000..63bce70 --- /dev/null +++ b/cyborg_enhancement/mitaka_version/cyborg/cyborg/accelerator/drivers/spdk/util/pyspdk/vhost_client.py @@ -0,0 +1,121 @@ +import json
+
+
+class VhostTgt(object):
+
+ def __init__(self, py):
+ super(VhostTgt, self).__init__()
+ self.py = py
+
+ def get_rpc_methods(self):
+ rpc_methods = self._get_json_objs('get_rpc_methods', '127.0.0.1')
+ return rpc_methods
+
+ def get_scsi_devices(self):
+ scsi_devices = self._get_json_objs(
+ 'get_scsi_devices', '127.0.0.1')
+ return scsi_devices
+
+ def get_luns(self):
+ luns = self._get_json_objs('get_luns', '127.0.0.1')
+ return luns
+
+ def get_interfaces(self):
+ interfaces = self._get_json_objs(
+ 'get_interfaces', '127.0.0.1')
+ return interfaces
+
+ def add_ip_address(self, ifc_index, ip_addr):
+ sub_args = [ifc_index, ip_addr]
+ res = self.py.exec_rpc(
+ 'add_ip_address',
+ '127.0.0.1',
+ sub_args=sub_args)
+ return res
+
+ def delete_ip_address(self, ifc_index, ip_addr):
+ sub_args = [ifc_index, ip_addr]
+ res = self.py.exec_rpc(
+ 'delete_ip_address',
+ '127.0.0.1',
+ sub_args=sub_args)
+ return res
+
+ def get_bdevs(self):
+ block_devices = self._get_json_objs(
+ 'get_bdevs', '127.0.0.1')
+ return block_devices
+
+ def delete_bdev(self, name):
+ sub_args = [name]
+ res = self.py.exec_rpc('delete_bdev', '127.0.0.1', sub_args=sub_args)
+ print res
+
+ def kill_instance(self, sig_name):
+ sub_args = [sig_name]
+ res = self.py.exec_rpc('kill_instance', '127.0.0.1', sub_args=sub_args)
+ print res
+
+ def construct_aio_bdev(self, filename, name, block_size):
+ sub_args = [filename, name, str(block_size)]
+ res = self.py.exec_rpc(
+ 'construct_aio_bdev',
+ '127.0.0.1',
+ sub_args=sub_args)
+ print res
+
+ def construct_error_bdev(self, basename):
+ sub_args = [basename]
+ res = self.py.exec_rpc(
+ 'construct_error_bdev',
+ '127.0.0.1',
+ sub_args=sub_args)
+ print res
+
+ def construct_nvme_bdev(
+ self,
+ name,
+ trtype,
+ traddr,
+ adrfam=None,
+ trsvcid=None,
+ subnqn=None):
+ sub_args = ["-b", "-t", "-a"]
+ sub_args.insert(1, name)
+ sub_args.insert(2, trtype)
+ sub_args.insert(3, traddr)
+ if adrfam is not None:
+ sub_args.append("-f")
+ sub_args.append(adrfam)
+ if trsvcid is not None:
+ sub_args.append("-s")
+ sub_args.append(trsvcid)
+ if subnqn is not None:
+ sub_args.append("-n")
+ sub_args.append(subnqn)
+ res = self.py.exec_rpc(
+ 'construct_nvme_bdev',
+ '127.0.0.1',
+ sub_args=sub_args)
+ return res
+
+ def construct_null_bdev(self, name, total_size, block_size):
+ sub_args = [name, str(total_size), str(block_size)]
+ res = self.py.exec_rpc(
+ 'construct_null_bdev',
+ '127.0.0.1',
+ sub_args=sub_args)
+ return res
+
+ def construct_malloc_bdev(self, total_size, block_size):
+ sub_args = [str(total_size), str(block_size)]
+ res = self.py.exec_rpc(
+ 'construct_malloc_bdev',
+ '10.0.2.15',
+ sub_args=sub_args)
+ print res
+
+ def _get_json_objs(self, method, server_ip):
+ res = self.py.exec_rpc(method, server_ip)
+ json_obj = json.loads(res)
+ return json_obj
\ No newline at end of file |