From: Yishai Hadas <yishaih@mellanox.com>
To: linux-rdma@vger.kernel.org
Cc: jgg@mellanox.com, yishaih@mellanox.com, maorg@mellanox.com,
Edward Srouji <edwards@mellanox.com>
Subject: [PATCH rdma-core 11/13] pyverbs: Support verbs import APIs
Date: Wed, 17 Jun 2020 10:45:54 +0300 [thread overview]
Message-ID: <1592379956-7043-12-git-send-email-yishaih@mellanox.com> (raw)
In-Reply-To: <1592379956-7043-1-git-send-email-yishaih@mellanox.com>
From: Edward Srouji <edwards@mellanox.com>
Importing a device, PD and MR enables processes to share their context
and then share PDs and MRs that is associated with.
This commit supports importing/unimporting a device, PD and MR by
wrapping the relevant verbs in the current Context, PD and MR objects.
Reviewed-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/device.pyx | 12 ++++++++--
pyverbs/libibverbs.pxd | 5 +++++
pyverbs/mr.pxd | 1 +
pyverbs/mr.pyx | 60 +++++++++++++++++++++++++++++++++++++++++---------
pyverbs/pd.pxd | 1 +
pyverbs/pd.pyx | 37 +++++++++++++++++++++++++------
6 files changed, 96 insertions(+), 20 deletions(-)
diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx
index 88cd906..95e112b 100755
--- a/pyverbs/device.pyx
+++ b/pyverbs/device.pyx
@@ -87,6 +87,9 @@ cdef class Context(PyverbsCM):
* *cmid*
A CMID object. If not None, it means that the device was already
opened by a CMID class, and only a pointer assignment is missing.
+ * *cmd_fd*
+ A command FD. If passed, the device will be imported from the
+ given cmd_fd using ibv_import_device.
:return: None
"""
cdef int count
@@ -107,10 +110,16 @@ cdef class Context(PyverbsCM):
self.name = kwargs.get('name')
provider_attr = kwargs.get('attr')
cmid = kwargs.get('cmid')
+ cmd_fd = kwargs.get('cmd_fd')
if cmid is not None:
self.context = cmid.id.verbs
cmid.ctx = self
return
+ if cmd_fd is not None:
+ self.context = v.ibv_import_device(cmd_fd)
+ if self.context == NULL:
+ raise PyverbsRDMAErrno('Failed to import device')
+ return
if self.name is None:
raise PyverbsUserError('Device name must be provided')
@@ -152,8 +161,7 @@ cdef class Context(PyverbsCM):
self.xrcds, self.vars])
rc = v.ibv_close_device(self.context)
if rc != 0:
- raise PyverbsRDMAErrno('Failed to close device {dev}'.
- format(dev=self.device.name))
+ raise PyverbsRDMAErrno(f'Failed to close device {self.name}')
self.context = NULL
@property
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index 4beb434..1aaf3f6 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -596,6 +596,11 @@ cdef extern from 'infiniband/verbs.h':
void ibv_wr_start(ibv_qp_ex *qp)
int ibv_wr_complete(ibv_qp_ex *qp)
void ibv_wr_abort(ibv_qp_ex *qp)
+ ibv_context *ibv_import_device(int cmd_fd)
+ ibv_mr *ibv_import_mr(ibv_pd *pd, uint32_t handle)
+ void ibv_unimport_mr(ibv_mr *mr)
+ ibv_pd *ibv_import_pd(ibv_context *context, uint32_t handle)
+ void ibv_unimport_pd(ibv_pd *pd)
cdef extern from 'infiniband/driver.h':
diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd
index 82ae79f..7c3bb8e 100644
--- a/pyverbs/mr.pxd
+++ b/pyverbs/mr.pxd
@@ -14,6 +14,7 @@ cdef class MR(PyverbsCM):
cdef object is_huge
cdef object is_user_addr
cdef void *buf
+ cdef object _is_imported
cpdef read(self, length, offset)
cdef class MWBindInfo(PyverbsCM):
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index b7b2196..da566cb 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -27,7 +27,7 @@ cdef class MR(PyverbsCM):
MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing
it is done in close().
"""
- def __init__(self, PD pd not None, length, access, address=None):
+ def __init__(self, PD pd not None, length=0, access=0, address=None, **kwargs):
"""
Allocate a user-level buffer of length <length> and register a Memory
Region of the given length and access flags.
@@ -37,6 +37,11 @@ cdef class MR(PyverbsCM):
:param address: Memory address to register (Optional). If it's not
provided, a memory will be allocated in the class
initialization.
+ :param kwargs: Arguments:
+ * *handle*
+ A valid kernel handle for a MR object in the given PD.
+ If passed, the MR will be imported and associated with the
+ context that is associated with the given PD using ibv_import_mr.
:return: The newly created MR on success
"""
super().__init__()
@@ -52,7 +57,20 @@ cdef class MR(PyverbsCM):
# uintptr_t is guaranteed to be large enough to hold any pointer.
# In order to safely cast addr to void*, it is firstly cast to uintptr_t.
self.buf = <void*><uintptr_t>address
- else:
+
+ mr_handle = kwargs.get('handle')
+ # If a MR handle is passed import MR and finish
+ if mr_handle is not None:
+ self.mr = v.ibv_import_mr(pd.pd, mr_handle)
+ if self.mr == NULL:
+ raise PyverbsRDMAErrno('Failed to import MR')
+ self._is_imported = True
+ self.pd = pd
+ pd.add_ref(self)
+ return
+
+ # Allocate a buffer
+ if not address:
if self.is_huge:
# Rounding up to multiple of HUGE_PAGE_SIZE
self.mmap_length = length + (HUGE_PAGE_SIZE - length % HUGE_PAGE_SIZE) \
@@ -77,6 +95,10 @@ cdef class MR(PyverbsCM):
self.logger.debug('Registered ibv_mr. Length: {l}, access flags {a}'.
format(l=length, a=access))
+ def unimport(self):
+ v.ibv_unimport_mr(self.mr)
+ self.close()
+
def __dealloc__(self):
self.close()
@@ -86,21 +108,24 @@ cdef class MR(PyverbsCM):
MR may be deleted directly or indirectly by closing its context, which
leaves the Python PD object without the underlying C object, so during
destruction, need to check whether or not the C object exists.
+ In case of an imported MR no deregistration will be done, it's left
+ for the original MR, in order to prevent double dereg by the GC.
:return: None
"""
if self.mr != NULL:
self.logger.debug('Closing MR')
- rc = v.ibv_dereg_mr(self.mr)
- if rc != 0:
- raise PyverbsRDMAError('Failed to dereg MR', rc)
+ if not self._is_imported:
+ rc = v.ibv_dereg_mr(self.mr)
+ if rc != 0:
+ raise PyverbsRDMAError('Failed to dereg MR', rc)
+ if not self.is_user_addr:
+ if self.is_huge:
+ munmap(self.buf, self.mmap_length)
+ else:
+ free(self.buf)
self.mr = NULL
self.pd = None
- if not self.is_user_addr:
- if self.is_huge:
- munmap(self.buf, self.mmap_length)
- else:
- free(self.buf)
- self.buf = NULL
+ self.buf = NULL
def write(self, data, length):
"""
@@ -144,6 +169,19 @@ cdef class MR(PyverbsCM):
def length(self):
return self.mr.length
+ @property
+ def handle(self):
+ return self.mr.handle
+
+ def __str__(self):
+ print_format = '{:22}: {:<20}\n'
+ return 'MR\n' + \
+ print_format.format('lkey', self.lkey) + \
+ print_format.format('rkey', self.rkey) + \
+ print_format.format('length', self.length) + \
+ print_format.format('buf', <uintptr_t>self.buf) + \
+ print_format.format('handle', self.handle)
+
cdef class MWBindInfo(PyverbsCM):
def __init__(self, MR mr not None, addr, length, mw_access_flags):
diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd
index ae4324a..94d453e 100644
--- a/pyverbs/pd.pxd
+++ b/pyverbs/pd.pxd
@@ -19,6 +19,7 @@ cdef class PD(PyverbsCM):
cdef object ahs
cdef object qps
cdef object parent_domains
+ cdef object _is_imported
cdef class ParentDomainInitAttr(PyverbsObject):
cdef v.ibv_parent_domain_init_attr init_attr
diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx
index d54c4f8..2a35d11 100755
--- a/pyverbs/pd.pyx
+++ b/pyverbs/pd.pyx
@@ -20,19 +20,31 @@ from pyverbs.qp cimport QP
cdef class PD(PyverbsCM):
- def __init__(self, object creator not None):
+ def __init__(self, object creator not None, **kwargs):
"""
Initializes a PD object. A reference for the creating Context is kept
so that Python's GC will destroy the objects in the right order.
:param creator: The Context/CMID object creating the PD
+ :param kwargs: Arguments:
+ * *handle*
+ A valid kernel handle for a PD object in the given creator
+ (Context). If passed, the PD will be imported and associated
+ with the given handle in the given context using ibv_import_pd.
"""
super().__init__()
+ pd_handle = kwargs.get('handle')
if issubclass(type(creator), Context):
# Check if the ibv_pd* was initialized by an inheriting class
if self.pd == NULL:
- self.pd = v.ibv_alloc_pd((<Context>creator).context)
+ if pd_handle is not None:
+ self.pd = v.ibv_import_pd((<Context>creator).context, pd_handle)
+ self._is_imported = True
+ err_str = 'Failed to import PD'
+ else:
+ self.pd = v.ibv_alloc_pd((<Context>creator).context)
+ err_str = 'Failed to allocate PD'
if self.pd == NULL:
- raise PyverbsRDMAErrno('Failed to allocate PD')
+ raise PyverbsRDMAErrno(err_str)
self.ctx = creator
elif issubclass(type(creator), CMID):
cmid = <CMID>creator
@@ -43,7 +55,7 @@ cdef class PD(PyverbsCM):
raise PyverbsUserError('Cannot create PD from {type}'
.format(type=type(creator)))
self.ctx.add_ref(self)
- self.logger.debug('PD: Allocated ibv_pd')
+ self.logger.debug('Created PD')
self.srqs = weakref.WeakSet()
self.mrs = weakref.WeakSet()
self.mws = weakref.WeakSet()
@@ -68,6 +80,10 @@ cdef class PD(PyverbsCM):
raise PyverbsRDMAError('Failed to advise MR', rc)
return rc
+ def unimport(self):
+ v.ibv_unimport_pd(self.pd)
+ self.close()
+
def __dealloc__(self):
"""
Closes the inner PD.
@@ -81,15 +97,18 @@ cdef class PD(PyverbsCM):
PD may be deleted directly or indirectly by closing its context, which
leaves the Python PD object without the underlying C object, so during
destruction, need to check whether or not the C object exists.
+ In case of an imported PD no deallocation will be done, it's left for
+ the original PD, in order to prevent double dealloc by the GC.
:return: None
"""
if self.pd != NULL:
self.logger.debug('Closing PD')
close_weakrefs([self.parent_domains, self.qps, self.ahs, self.mws,
self.mrs, self.srqs])
- rc = v.ibv_dealloc_pd(self.pd)
- if rc != 0:
- raise PyverbsRDMAError('Failed to dealloc PD', rc)
+ if not self._is_imported:
+ rc = v.ibv_dealloc_pd(self.pd)
+ if rc != 0:
+ raise PyverbsRDMAError('Failed to dealloc PD', rc)
self.pd = NULL
self.ctx = None
@@ -109,6 +128,10 @@ cdef class PD(PyverbsCM):
else:
raise PyverbsError('Unrecognized object type')
+ @property
+ def handle(self):
+ return self.pd.handle
+
cdef void *pd_alloc(v.ibv_pd *pd, void *pd_context, size_t size,
size_t alignment, v.uint64_t resource_type):
--
1.8.3.1
next prev parent reply other threads:[~2020-06-17 7:46 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-06-17 7:45 [PATCH rdma-core 00/13] verbs: Introduce import verbs for device, PD, MR Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 01/13] Update kernel headers Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 02/13] verbs: Close async_fd only when it was previously created Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 03/13] verbs: Introduce ibv_import_device() verb Yishai Hadas
2020-06-19 12:29 ` Jason Gunthorpe
2020-06-21 7:01 ` Yishai Hadas
2020-06-22 12:52 ` Jason Gunthorpe
2020-06-23 13:06 ` Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 04/13] verbs: Handle async FD on an imported device Yishai Hadas
2020-06-19 12:33 ` Jason Gunthorpe
2020-06-21 9:08 ` Yishai Hadas
2020-06-23 17:34 ` Jason Gunthorpe
2020-06-24 7:22 ` Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 05/13] mlx5: Refactor mlx5_alloc_context() Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 06/13] mlx5: Implement the import device functionality Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 07/13] verbs: Introduce ibv_import/unimport_pd() verbs Yishai Hadas
2020-06-19 12:48 ` Jason Gunthorpe
2020-06-21 8:30 ` Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 08/13] mlx5: Implement the import/unimport PD verbs Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 09/13] verbs: Introduce ibv_import/unimport_mr() verbs Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 10/13] mlx5: Implement the import/unimport MR verbs Yishai Hadas
2020-06-19 12:50 ` Jason Gunthorpe
2020-06-21 8:44 ` Yishai Hadas
2020-06-23 17:33 ` Jason Gunthorpe
2020-06-24 7:20 ` Yishai Hadas
2020-07-01 12:28 ` Yishai Hadas
2020-07-02 17:42 ` Jason Gunthorpe
2020-06-17 7:45 ` Yishai Hadas [this message]
2020-06-17 7:45 ` [PATCH rdma-core 12/13] Documentation: Add usage example for verbs import Yishai Hadas
2020-06-17 7:45 ` [PATCH rdma-core 13/13] tests: Add a shared PD Pyverbs test Yishai Hadas
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1592379956-7043-12-git-send-email-yishaih@mellanox.com \
--to=yishaih@mellanox.com \
--cc=edwards@mellanox.com \
--cc=jgg@mellanox.com \
--cc=linux-rdma@vger.kernel.org \
--cc=maorg@mellanox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).