* [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API
@ 2019-12-31 9:19 Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
` (7 more replies)
0 siblings, 8 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Noa Osherovich
The following series adds pyverbs support for the new post send API.
The first 5 patches add unrelated but needed support (e.g. memory
window) and fixes for bugs that were found along the way.
These patches are followed by the feature itself, a documentation and
a test.
Noa Osherovich (8):
pyverbs: Add support for memory window
pyverbs: Add TSO support
tests: Decrease maximal TSO header size
tests: Use post_recv in the right place
pyverbs: Expose MR's length property
pyverbs: Introduce extended QP and new post send
Documentation: Add extended QP to pyverbs's doc
tests: Add test using the new post send API
Documentation/pyverbs.md | 24 +++
pyverbs/base.pyx | 8 +-
pyverbs/libibverbs.pxd | 48 +++++-
pyverbs/libibverbs_enums.pxd | 14 ++
pyverbs/mr.pxd | 6 +
pyverbs/mr.pyx | 23 +++
pyverbs/qp.pxd | 6 +
pyverbs/qp.pyx | 154 +++++++++++++++++-
pyverbs/wr.pxd | 6 +
pyverbs/wr.pyx | 60 ++++++-
tests/CMakeLists.txt | 1 +
tests/test_odp.py | 13 +-
tests/test_qpex.py | 295 +++++++++++++++++++++++++++++++++++
tests/utils.py | 213 +++++++++++++++++++++----
14 files changed, 817 insertions(+), 54 deletions(-)
mode change 100755 => 100644 tests/CMakeLists.txt
create mode 100644 tests/test_qpex.py
--
2.21.0
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH rdma-core 1/8] pyverbs: Add support for memory window
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 2/8] pyverbs: Add TSO support Noa Osherovich
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Ahmad Ghazawi, Edward Srouji
In order to use memory window, some additions were needed:
- Allow pyverbs users to create MWBindInfo, the Python representation
of ibv_mw_bind_info.
- Expose ibv_inc_rkey, which creates a new rkey from a given one by
increasing its 8 LSBs while keeping the same index.
- Expose the memory window's rkey and handle properties.
Signed-off-by: Ahmad Ghazawi <ahmadg@mellanox.com>
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/base.pyx | 8 +++++++-
pyverbs/libibverbs.pxd | 1 +
pyverbs/mr.pxd | 6 ++++++
pyverbs/mr.pyx | 19 +++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx
index c5b16795ddb6..790ba4153dea 100644
--- a/pyverbs/base.pyx
+++ b/pyverbs/base.pyx
@@ -1,9 +1,15 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from libc.errno cimport errno
import logging
+
from pyverbs.pyverbs_error import PyverbsRDMAError
-from libc.errno cimport errno
+cimport pyverbs.libibverbs as v
+
+
+def inc_rkey(rkey):
+ return v.ibv_inc_rkey(rkey)
cpdef PyverbsRDMAErrno(str msg):
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index ad8d8bacc541..fea8a1b408da 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -551,3 +551,4 @@ cdef extern from 'infiniband/verbs.h':
ibv_recv_wr **bad_recv_wr)
ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
ibv_parent_domain_init_attr *attr)
+ uint32_t ibv_inc_rkey(uint32_t rkey)
diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd
index fb46611e6f42..402df4492425 100644
--- a/pyverbs/mr.pxd
+++ b/pyverbs/mr.pxd
@@ -12,6 +12,12 @@ cdef class MR(PyverbsCM):
cdef v.ibv_mr *mr
cdef void *buf
cpdef read(self, length, offset)
+ cdef add_ref(self, obj)
+ cdef object bind_infos
+
+cdef class MWBindInfo(PyverbsCM):
+ cdef v.ibv_mw_bind_info info
+ cdef object mr
cdef class MW(PyverbsCM):
cdef object pd
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 6b28c8173ef8..9b1277f0882f 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -3,10 +3,12 @@
import resource
import logging
+import weakref
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
from posix.stdlib cimport posix_memalign
+from pyverbs.base cimport close_weakrefs
from libc.string cimport memcpy, memset
from libc.stdint cimport uintptr_t
from pyverbs.device cimport DM
@@ -29,6 +31,7 @@ cdef class MR(PyverbsCM):
:return: The newly created MR on success
"""
super().__init__()
+ self.bind_infos = weakref.WeakSet()
if self.mr != NULL:
return
#We want to enable registering an MR of size 0 but this fails with a
@@ -61,6 +64,7 @@ cdef class MR(PyverbsCM):
:return: None
"""
self.logger.debug('Closing MR')
+ close_weakrefs([self.bind_infos])
if self.mr != NULL:
rc = v.ibv_dereg_mr(self.mr)
if rc != 0:
@@ -96,6 +100,10 @@ cdef class MR(PyverbsCM):
data = <char*>(self.buf + off)
return data[:length]
+ cdef add_ref(self, obj):
+ if isinstance(obj, MWBindInfo):
+ self.bind_infos.add(obj)
+
@property
def buf(self):
return <uintptr_t>self.buf
@@ -109,6 +117,17 @@ cdef class MR(PyverbsCM):
return self.mr.rkey
+cdef class MWBindInfo(PyverbsCM):
+ def __init__(self, MR mr not None, addr, length, mw_access_flags):
+ super().__init__()
+ self.mr = mr
+ self.info.mr = mr.mr
+ self.info.addr = addr
+ self.info.length = length
+ self.info.mw_access_flags = mw_access_flags
+ mr.add_ref(self)
+
+
cdef class MW(PyverbsCM):
def __init__(self, PD pd not None, v.ibv_mw_type mw_type):
"""
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 2/8] pyverbs: Add TSO support
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size Noa Osherovich
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Ahmad Ghazawi, Edward Srouji
Add TSO class to allow users to easily set the TSO field in a send
work request.
Signed-off-by: Ahmad Ghazawi <ahmadg@mellanox.com>
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/libibverbs.pxd | 7 ++---
pyverbs/wr.pxd | 6 +++++
pyverbs/wr.pyx | 60 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 66 insertions(+), 7 deletions(-)
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index fea8a1b408da..8949759511a5 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -311,10 +311,6 @@ cdef extern from 'infiniband/verbs.h':
unsigned short hdr_sz
unsigned short mss
- cdef union unnamed:
- bind_mw bind_mw
- tso tso
-
cdef struct xrc:
unsigned int remote_srqn
@@ -330,7 +326,8 @@ cdef extern from 'infiniband/verbs.h':
unsigned int send_flags
wr wr
qp_type qp_type
- unnamed unnamed
+ bind_mw bind_mw
+ tso tso
cdef struct ibv_qp_cap:
unsigned int max_send_wr
diff --git a/pyverbs/wr.pxd b/pyverbs/wr.pxd
index e259249ef7f8..5cb282dc65c3 100644
--- a/pyverbs/wr.pxd
+++ b/pyverbs/wr.pxd
@@ -16,3 +16,9 @@ cdef class RecvWR(PyverbsCM):
cdef class SendWR(PyverbsCM):
cdef v.ibv_send_wr send_wr
+
+cdef class TSO(PyverbsCM):
+ cdef void* buf
+ cdef int length
+ cdef int mss
+ cpdef alloc_buf(self, content, length)
diff --git a/pyverbs/wr.pyx b/pyverbs/wr.pyx
index 06186e9b0fd2..b1df3677fce3 100644
--- a/pyverbs/wr.pyx
+++ b/pyverbs/wr.pyx
@@ -1,12 +1,16 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies Inc. All rights reserved. See COPYING file
+import resource
+
+from cpython.pycapsule cimport PyCapsule_New
+from libc.stdlib cimport free, malloc
+from libc.string cimport memcpy
+
from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
from pyverbs.addr cimport AH
-from libc.stdlib cimport free, malloc
-from libc.string cimport memcpy
cdef class SGE(PyverbsCM):
@@ -281,6 +285,58 @@ cdef class SendWR(PyverbsCM):
"""
self.send_wr.qp_type.xrc.remote_srqn = remote_srqn
+ def set_tso(self, TSO tso not None):
+ """
+ Set the members of the tso struct in the send_wr's anonymous union.
+ :param tso: A TSO object to copy to the work request
+ :return: None
+ """
+ self.send_wr.tso.hdr = tso.buf
+ self.send_wr.tso.hdr_sz = tso.length
+ self.send_wr.tso.mss = tso.mss
+
+
+cdef class TSO(PyverbsCM):
+ """ Represents the 'tso' anonymous struct inside a send WR """
+ def __init__(self, data, length, mss):
+ super().__init__()
+ self.alloc_buf(data, length)
+ self.length = length
+ self.mss = mss
+
+ cpdef alloc_buf(self, data, length):
+ if self.buf != NULL:
+ free(self.buf)
+ self.buf = malloc(length)
+ if self.buf == NULL:
+ raise PyverbsError('Failed to allocate TSO buffer of length {l}'.\
+ format(l=length))
+ if isinstance(data, str):
+ data = data.encode()
+ memcpy(self.buf, <char*>data, length)
+ self.length = length
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ if self.buf != NULL:
+ free(self.buf)
+ self.buf = NULL
+
+ @property
+ def buf(self):
+ return PyCapsule_New(self.buf, NULL, NULL)
+
+ def __str__(self):
+ print_format = '{:22}: {:<20}\n'
+ data = <char*>(self.buf)
+ data = data[:self.length].decode()
+ return print_format.format('MSS', self.mss) +\
+ print_format.format('Length', self.length) +\
+ print_format.format('Data', data)
+
+
def send_flags_to_str(flags):
send_flags = {e.IBV_SEND_FENCE: 'IBV_SEND_FENCE',
e.IBV_SEND_SIGNALED: 'IBV_SEND_SIGNALED',
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 2/8] pyverbs: Add TSO support Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 4/8] tests: Use post_recv in the right place Noa Osherovich
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
When creating a Raw Packet QP with a TSO header, send WQE size
increases to accommodate it. If both max_tso_header and max_send_wr
are too large, the requested WQE size can become too large for the
device to support.
Decrease max_tso_header's max value to avoid that.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
tests/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/utils.py b/tests/utils.py
index c45170dbd329..d4d0d1ef49ef 100755
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -224,7 +224,7 @@ def random_qp_init_attr_ex(attr_ex, attr, qpt=None):
mask -= e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER
else:
max_tso = \
- random.randint(16, int(attr_ex.tso_caps.max_tso / 400))
+ random.randint(16, int(attr_ex.tso_caps.max_tso / 800))
qia = QPInitAttrEx(qp_type=qpt, cap=qp_cap, sq_sig_all=sig, comp_mask=mask,
create_flags=cflags, max_tso_header=max_tso)
if mask & e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER:
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 4/8] tests: Use post_recv in the right place
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (2 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 5/8] pyverbs: Expose MR's length property Noa Osherovich
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
In traffic() helper method, do post_recv after a recv WQE was consumed
by the hardware.
Fixes: 6fb2b9bade55 ('tests: Add traffic helper methods')
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
tests/utils.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/utils.py b/tests/utils.py
index d4d0d1ef49ef..47eacfee35e5 100755
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -420,14 +420,14 @@ def traffic(client, server, iters, gid_idx, port, is_cq_ex=False):
post_send(client, c_send_wr, gid_idx, port)
poll(client.cq)
poll(server.cq)
- post_recv(client.qp, c_recv_wr)
+ post_recv(server.qp, s_recv_wr)
msg_received = server.mr.read(server.msg_size, read_offset)
validate(msg_received, True, server.msg_size)
s_send_wr = get_send_wr(server, True)
post_send(server, s_send_wr, gid_idx, port)
poll(server.cq)
poll(client.cq)
- post_recv(server.qp, s_recv_wr)
+ post_recv(client.qp, c_recv_wr)
msg_received = client.mr.read(client.msg_size, read_offset)
validate(msg_received, False, client.msg_size)
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 5/8] pyverbs: Expose MR's length property
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (3 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 4/8] tests: Use post_recv in the right place Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send Noa Osherovich
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
MR's length is needed for testing purposes.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/mr.pyx | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 9b1277f0882f..234e864209a2 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -116,6 +116,10 @@ cdef class MR(PyverbsCM):
def rkey(self):
return self.mr.rkey
+ @property
+ def length(self):
+ return self.mr.length
+
cdef class MWBindInfo(PyverbsCM):
def __init__(self, MR mr not None, addr, length, mw_access_flags):
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (4 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 5/8] pyverbs: Expose MR's length property Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 8/8] tests: Add test using the new post send API Noa Osherovich
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Ahmad Ghazawi, Edward Srouji
Add QPEx class alongside the new post send functions.
The new post send flow is as follows:
- Create a WR on the extended QP.
- Set the WR's attributes.
- Commit the entries in the send buffer and ring the doorbell.
QPEx class exposes the relevant methods (wr_*) for that.
Signed-off-by: Ahmad Ghazawi <ahmadg@mellanox.com>
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/libibverbs.pxd | 40 +++++++++
pyverbs/libibverbs_enums.pxd | 14 ++++
pyverbs/qp.pxd | 6 ++
pyverbs/qp.pyx | 154 +++++++++++++++++++++++++++++++++--
4 files changed, 209 insertions(+), 5 deletions(-)
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index 8949759511a5..c038c4aa4d5b 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -438,6 +438,10 @@ cdef extern from 'infiniband/verbs.h':
unsigned int handle
unsigned int events_completed
+ cdef struct ibv_data_buf:
+ void *addr
+ size_t length
+
cdef struct ibv_qp:
ibv_context *context;
void *qp_context;
@@ -460,6 +464,12 @@ cdef extern from 'infiniband/verbs.h':
uint64_t resource_type);
void *pd_context;
+ cdef struct ibv_qp_ex:
+ ibv_qp qp_base
+ uint64_t comp_mask
+ uint64_t wr_id
+ unsigned int wr_flags
+
ibv_device **ibv_get_device_list(int *n)
void ibv_free_device_list(ibv_device **list)
ibv_context *ibv_open_device(ibv_device *device)
@@ -549,3 +559,33 @@ cdef extern from 'infiniband/verbs.h':
ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
ibv_parent_domain_init_attr *attr)
uint32_t ibv_inc_rkey(uint32_t rkey)
+ ibv_qp_ex *ibv_qp_to_qp_ex(ibv_qp *qp)
+ void ibv_wr_atomic_cmp_swp(ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t compare,
+ uint64_t swap)
+ void ibv_wr_atomic_fetch_add(ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t add)
+ void ibv_wr_bind_mw(ibv_qp_ex *qp, ibv_mw *mw, uint32_t rkey,
+ ibv_mw_bind_info *bind_info)
+ void ibv_wr_local_inv(ibv_qp_ex *qp, uint32_t invalidate_rkey)
+ void ibv_wr_rdma_read(ibv_qp_ex *qp, uint32_t rkey, uint64_t remote_addr)
+ void ibv_wr_rdma_write(ibv_qp_ex *qp, uint32_t rkey, uint64_t remote_addr)
+ void ibv_wr_rdma_write_imm(ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint32_t imm_data)
+ void ibv_wr_send(ibv_qp_ex *qp)
+ void ibv_wr_send_imm(ibv_qp_ex *qp, uint32_t imm_data)
+ void ibv_wr_send_inv(ibv_qp_ex *qp, uint32_t invalidate_rkey)
+ void ibv_wr_send_tso(ibv_qp_ex *qp, void *hdr, uint16_t hdr_sz,
+ uint16_t mss)
+ void ibv_wr_set_ud_addr(ibv_qp_ex *qp, ibv_ah *ah, uint32_t remote_qpn,
+ uint32_t remote_qkey)
+ void ibv_wr_set_xrc_srqn(ibv_qp_ex *qp, uint32_t remote_srqn)
+ void ibv_wr_set_inline_data(ibv_qp_ex *qp, void *addr, size_t length)
+ void ibv_wr_set_inline_data_list(ibv_qp_ex *qp, size_t num_buf,
+ ibv_data_buf *buf_list)
+ void ibv_wr_set_sge(ibv_qp_ex *qp, uint32_t lkey, uint64_t addr,
+ uint32_t length)
+ void ibv_wr_set_sge_list(ibv_qp_ex *qp, size_t num_sge, ibv_sge *sg_list)
+ void ibv_wr_start(ibv_qp_ex *qp)
+ int ibv_wr_complete(ibv_qp_ex *qp)
+ void ibv_wr_abort(ibv_qp_ex *qp)
diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd
index fd6a6f49a163..a706cec0aba8 100755
--- a/pyverbs/libibverbs_enums.pxd
+++ b/pyverbs/libibverbs_enums.pxd
@@ -236,6 +236,7 @@ cdef extern from '<infiniband/verbs.h>':
IBV_QP_INIT_ATTR_MAX_TSO_HEADER
IBV_QP_INIT_ATTR_IND_TABLE
IBV_QP_INIT_ATTR_RX_HASH
+ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
cpdef enum ibv_qp_create_flags:
IBV_QP_CREATE_BLOCK_SELF_MCAST_LB
@@ -401,6 +402,19 @@ cdef extern from '<infiniband/verbs.h>':
cpdef enum:
IBV_WC_STANDARD_FLAGS
+ cpdef enum ibv_qp_create_send_ops_flags:
+ IBV_QP_EX_WITH_RDMA_WRITE
+ IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM
+ IBV_QP_EX_WITH_SEND
+ IBV_QP_EX_WITH_SEND_WITH_IMM
+ IBV_QP_EX_WITH_RDMA_READ
+ IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP
+ IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD
+ IBV_QP_EX_WITH_LOCAL_INV
+ IBV_QP_EX_WITH_BIND_MW
+ IBV_QP_EX_WITH_SEND_WITH_INV
+ IBV_QP_EX_WITH_TSO
+
cdef unsigned long long IBV_DEVICE_RAW_SCATTER_FCS
cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING
diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd
index 52aab503e40d..209a2438dd83 100644
--- a/pyverbs/qp.pxd
+++ b/pyverbs/qp.pxd
@@ -37,3 +37,9 @@ cdef class QP(PyverbsCM):
cdef update_cqs(self, init_attr)
cdef object scq
cdef object rcq
+
+cdef class DataBuffer(PyverbsCM):
+ cdef v.ibv_data_buf data
+
+cdef class QPEx(QP):
+ cdef v.ibv_qp_ex *qp_ex
diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx
index 9d368b62022d..1fcb23909758 100755
--- a/pyverbs/qp.pyx
+++ b/pyverbs/qp.pyx
@@ -1,20 +1,30 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memcpy
+
from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
from pyverbs.utils import access_flags_to_str, mig_state_to_str
-from pyverbs.pyverbs_error import PyverbsUserError
+from pyverbs.wr cimport RecvWR, SendWR, SGE
from pyverbs.base import PyverbsRDMAErrno
-from pyverbs.wr cimport RecvWR, SendWR
+from pyverbs.addr cimport AHAttr, GID, AH
+from pyverbs.mr cimport MW, MWBindInfo
cimport pyverbs.libibverbs_enums as e
-from pyverbs.addr cimport AHAttr, GID
from pyverbs.addr cimport GlobalRoute
from pyverbs.device cimport Context
+from cpython.ref cimport PyObject
from pyverbs.cq cimport CQ, CQEX
cimport pyverbs.libibverbs as v
from pyverbs.xrcd cimport XRCD
from pyverbs.srq cimport SRQ
from pyverbs.pd cimport PD
-from libc.string cimport memcpy
+
+cdef extern from 'Python.h':
+ void* PyLong_AsVoidPtr(object)
+cdef extern from 'endian.h':
+ unsigned long htobe32(unsigned long host_32bits)
cdef class QPCap(PyverbsObject):
@@ -240,7 +250,7 @@ cdef class QPInitAttrEx(PyverbsObject):
SRQ srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0,
PD pd=None, XRCD xrcd=None, create_flags=0,
max_tso_header=0, source_qpn=0, object hash_conf=None,
- object ind_table=None):
+ object ind_table=None, send_ops_flags=0):
"""
Initialize a QPInitAttrEx object with user-defined or default values.
:param qp_type: QP type to be created
@@ -261,6 +271,8 @@ cdef class QPInitAttrEx(PyverbsObject):
set in create_flags)
:param hash_conf: Not yet supported
:param ind_table: Not yet supported
+ :param send_ops_flags: Send opcodes to be supported by the extended QP.
+ Use ibv_qp_create_send_ops_flags enum
:return: An initialized QPInitAttrEx object
"""
super().__init__()
@@ -302,6 +314,7 @@ cdef class QPInitAttrEx(PyverbsObject):
self.attr.create_flags = create_flags
self.attr.max_tso_header = max_tso_header
self.attr.source_qpn = source_qpn
+ self.attr.send_ops_flags = send_ops_flags
@property
def send_cq(self):
@@ -1141,6 +1154,137 @@ cdef class QP(PyverbsCM):
print_format.format(' state', qp_state_to_str(self.qp_state))
+cdef class DataBuffer(PyverbsCM):
+ def __init__(self, addr, length):
+ super().__init__()
+ self.data.addr = PyLong_AsVoidPtr(addr)
+ self.data.length = length
+
+
+cdef class QPEx(QP):
+ def __init__(self, object creator not None, object init_attr not None,
+ QPAttr qp_attr=None):
+ """
+ Initializes a QPEx object. Since this is an extension of a QP, QP
+ creation is done in the parent class. The extended QP is retrieved by
+ casting the ibv_qp to ibv_qp_ex.
+ :return: An initialized QPEx object
+ """
+ super().__init__(creator, init_attr, qp_attr)
+ self.qp_ex = v.ibv_qp_to_qp_ex(self.qp)
+ if self.qp_ex == NULL:
+ raise PyverbsRDMAErrno('Failed to create extended QP')
+
+ @property
+ def comp_mask(self):
+ return self.qp_ex.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.qp_ex.comp_mask = val
+
+ @property
+ def wr_id(self):
+ return self.qp_ex.wr_id
+ @wr_id.setter
+ def wr_id(self, val):
+ self.qp_ex.wr_id = val
+
+ @property
+ def wr_flags(self):
+ return self.qp_ex.wr_flags
+ @wr_flags.setter
+ def wr_flags(self, val):
+ self.qp_ex.wr_flags = val
+
+ def wr_atomic_cmp_swp(self, rkey, remote_addr, compare, swap):
+ v.ibv_wr_atomic_cmp_swp(self.qp_ex, rkey, remote_addr, compare, swap)
+
+ def wr_atomic_fetch_add(self, rkey, remote_addr, add):
+ v.ibv_wr_atomic_fetch_add(self.qp_ex, rkey, remote_addr, add)
+
+ def wr_bind_mw(self, MW mw, rkey, MWBindInfo bind_info):
+ cdef v.ibv_mw_bind_info *info
+ info = &bind_info.info
+ v.ibv_wr_bind_mw(self.qp_ex, <v.ibv_mw*>mw.mw, rkey,
+ <v.ibv_mw_bind_info*>info)
+
+ def wr_local_inv(self, invalidate_rkey):
+ v.ibv_wr_local_inv(self.qp_ex, invalidate_rkey)
+
+ def wr_rdma_read(self, rkey, remote_addr):
+ v.ibv_wr_rdma_read(self.qp_ex, rkey, remote_addr)
+
+ def wr_rdma_write(self, rkey, remote_addr):
+ v.ibv_wr_rdma_write(self.qp_ex, rkey, remote_addr)
+
+ def wr_rdma_write_imm(self, rkey, remote_addr, data):
+ cdef unsigned int imm_data = htobe32(data)
+ v.ibv_wr_rdma_write_imm(self.qp_ex, rkey, remote_addr, imm_data)
+
+ def wr_send(self):
+ v.ibv_wr_send(self.qp_ex)
+
+ def wr_send_imm(self, data):
+ cdef unsigned int imm_data = htobe32(data)
+ return v.ibv_wr_send_imm(self.qp_ex, imm_data)
+
+ def wr_send_inv(self, invalidate_rkey):
+ v.ibv_wr_send_inv(self.qp_ex, invalidate_rkey)
+
+ def wr_send_tso(self, hdr, hdr_sz, mss):
+ ptr = PyLong_AsVoidPtr(hdr)
+ v.ibv_wr_send_tso(self.qp_ex, ptr, hdr_sz, mss)
+
+ def wr_set_ud_addr(self, AH ah, remote_qpn, remote_rkey):
+ v.ibv_wr_set_ud_addr(self.qp_ex, ah.ah, remote_qpn, remote_rkey)
+
+ def wr_set_xrc_srqn(self, remote_srqn):
+ v.ibv_wr_set_xrc_srqn(self.qp_ex, remote_srqn)
+
+ def wr_set_inline_data(self, addr, length):
+ ptr = PyLong_AsVoidPtr(addr)
+ v.ibv_wr_set_inline_data(self.qp_ex, ptr, length)
+
+ def wr_set_inline_data_list(self, num_buf, buf_list):
+ cdef v.ibv_data_buf *data = NULL
+ data = <v.ibv_data_buf*>malloc(num_buf * sizeof(v.ibv_data_buf))
+ if data == NULL:
+ raise PyverbsError('Failed to allocate data buffer')
+ for i in range(num_buf):
+ data_buf = <DataBuffer>buf_list[i]
+ data[i].addr = data_buf.data.addr
+ data[i].length = data_buf.data.length
+ v.ibv_wr_set_inline_data_list(self.qp_ex, num_buf, data)
+ free(data)
+
+ def wr_set_sge(self, SGE sge not None):
+ v.ibv_wr_set_sge(self.qp_ex, sge.lkey, sge.addr, sge.length)
+
+ def wr_set_sge_list(self, num_sge, sg_list):
+ cdef v.ibv_sge *sge = NULL
+ sge = <v.ibv_sge*>malloc(num_sge * sizeof(v.ibv_sge))
+ if sge == NULL:
+ raise PyverbsError('Failed to allocate SGE buffer')
+ for i in range(num_sge):
+ sge[i].addr = sg_list[i].addr
+ sge[i].length = sg_list[i].length
+ sge[i].lkey = sg_list[i].lkey
+ v.ibv_wr_set_sge_list(self.qp_ex, num_sge, sge)
+ free(sge)
+
+ def wr_start(self):
+ v.ibv_wr_start(self.qp_ex)
+
+ def wr_complete(self):
+ rc = v.ibv_wr_complete(self.qp_ex)
+ if rc != 0:
+ raise PyverbsRDMAErrno('ibv_wr_complete failed , returned {}'.
+ format(rc))
+
+ def wr_abort(self):
+ v.ibv_wr_abort(self.qp_ex)
+
+
def _copy_caps(QPCap src, dst):
"""
Copy the QPCaps values of src into the inner ibv_qp_cap struct of dst.
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (5 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 8/8] tests: Add test using the new post send API Noa Osherovich
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
Add a code snippet to demonstrate an extended QP creation using
pyverbs.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
Documentation/pyverbs.md | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md
index c7dddd598303..325e214fca0e 100755
--- a/Documentation/pyverbs.md
+++ b/Documentation/pyverbs.md
@@ -339,6 +339,30 @@ wr = pwr.SendWR()
wr.set_wr_ud(ah, 0x1101, 0) # in real life, use real values
udqp.post_send(wr)
```
+###### Extended QP
+An extended QP exposes a new set of QP send operations to the user -
+extensibility for new send opcodes, vendor specific send opcodes and even vendor
+specific QP types.
+Pyverbs now exposes the needed interface to create such a QP.
+Note that the IBV_QP_INIT_ATTR_SEND_OPS_FLAGS in the `comp_mask` is mandatory
+when using the extended QP's new post send mechanism.
+```python
+from pyverbs.qp import QPCap, QPInitAttrEx, QPAttr, QPEx
+import pyverbs.device as d
+import pyverbs.enums as e
+from pyverbs.pd import PD
+from pyverbs.cq import CQ
+
+
+ctx = d.Context(name='mlx5_0')
+pd = PD(ctx)
+cq = CQ(ctx, 100)
+cap = QPCap(100, 10, 1, 1, 0)
+qia = QPInitAttrEx(qp_type=e.IBV_QPT_UD, scq=cq, rcq=cq, cap=cap, pd=pd,
+ comp_mask=e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS| \
+ e.IBV_QP_INIT_ATTR_PD)
+qp = QPEx(ctx, qia)
+```
##### XRCD
The following code demonstrates creation of an XRCD object.
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 8/8] tests: Add test using the new post send API
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (6 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
Add simple traffic tests that use the new post_send API.
Currently tested include:
- UD: send, send with immediate
- RC: send, send with immediate, RDMA write, RDMA read, atomic fetch
and add, atomic compare and swap, bind memory window
- XRC: send, send with immediate
The existing traffic methods - traffic() and xrc_traffic() were
modified to support usage of the new API and are now checking whether
a send_op was provided or not.
As RDMA read and write do not require receive WQEs to be posted, an
extra traffic method, rdma_traffic, was added to the tests' utils
section.
Creation of a customized memory region is now available in the tests'
utils section.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
tests/CMakeLists.txt | 1 +
tests/test_odp.py | 13 +-
tests/test_qpex.py | 295 +++++++++++++++++++++++++++++++++++++++++++
tests/utils.py | 207 +++++++++++++++++++++++++-----
4 files changed, 478 insertions(+), 38 deletions(-)
mode change 100755 => 100644 tests/CMakeLists.txt
create mode 100644 tests/test_qpex.py
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
old mode 100755
new mode 100644
index 6d702425886c..74930f69e19d
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -12,6 +12,7 @@ rdma_python_test(tests
test_mr.py
test_pd.py
test_qp.py
+ test_qpex.py
test_odp.py
test_parent_domain.py
test_rdmacm.py
diff --git a/tests/test_odp.py b/tests/test_odp.py
index d412a7792951..742ad81abb89 100755
--- a/tests/test_odp.py
+++ b/tests/test_odp.py
@@ -1,5 +1,5 @@
+from tests.utils import requires_odp, traffic, xrc_traffic, create_custom_mr
from tests.base import RCResources, UDResources, XRCResources
-from tests.utils import requires_odp, traffic, xrc_traffic
from tests.base import RDMATestCase
from pyverbs.mr import MR
import pyverbs.enums as e
@@ -8,21 +8,20 @@ import pyverbs.enums as e
class OdpUD(UDResources):
@requires_odp('ud')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE,
- e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+ self.mr = create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND,
+ self.msg_size + self.GRH_SIZE)
class OdpRC(RCResources):
@requires_odp('rc')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size,
- e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+ self.mr = create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND)
+
class OdpXRC(XRCResources):
@requires_odp('xrc')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size,
- e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+ self.mr = create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND)
class OdpTestCase(RDMATestCase):
diff --git a/tests/test_qpex.py b/tests/test_qpex.py
new file mode 100644
index 000000000000..922010bce3e5
--- /dev/null
+++ b/tests/test_qpex.py
@@ -0,0 +1,295 @@
+import unittest
+import random
+
+from pyverbs.qp import QPCap, QPInitAttrEx, QPAttr, QPEx, QP
+from pyverbs.pyverbs_error import PyverbsRDMAError
+from pyverbs.mr import MW, MWBindInfo
+from pyverbs.base import inc_rkey
+import pyverbs.enums as e
+from pyverbs.mr import MR
+
+from tests.base import UDResources, RCResources, RDMATestCase, XRCResources
+import tests.utils as u
+
+
+def create_qp_ex(agr_obj, qp_type, send_flags):
+ if qp_type == e.IBV_QPT_XRC_SEND:
+ cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0,
+ max_send_sge=1)
+ else:
+ cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs,
+ max_recv_sge=1, max_send_sge=1)
+ qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq,
+ rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags,
+ comp_mask=e.IBV_QP_INIT_ATTR_PD |
+ e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
+ qp_attr = QPAttr(port_num=agr_obj.ib_port)
+ if qp_type == e.IBV_QPT_UD:
+ qp_attr.qkey = agr_obj.UD_QKEY
+ qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
+ if qp_type == e.IBV_QPT_RC:
+ qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+ e.IBV_ACCESS_REMOTE_READ | \
+ e.IBV_ACCESS_REMOTE_ATOMIC
+ try:
+ # We don't have capability bits for this
+ qp = QPEx(agr_obj.ctx, qia, qp_attr)
+ except PyverbsRDMAError as exp:
+ if 'Operation not supported' in exp.args[0]:
+ raise unittest.SkipTest('Extended QP not supported on this device')
+ raise exp
+ return qp
+
+
+class QpExUDSend(UDResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND)
+
+
+class QpExRCSend(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND)
+
+
+class QpExXRCSend(XRCResources):
+ def create_qp(self):
+ qp_attr = QPAttr(port_num=self.ib_port)
+ qp_attr.pkey_index = 0
+ for _ in range(self.qp_count):
+ attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
+ comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
+ xrcd=self.xrcd)
+ qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+ e.IBV_ACCESS_REMOTE_READ
+ recv_qp = QP(self.ctx, attr_ex, qp_attr)
+ self.rqp_lst.append(recv_qp)
+
+ send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND, e.IBV_QP_EX_WITH_SEND)
+ self.sqp_lst.append(send_qp)
+ self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
+ self.psns.append(random.getrandbits(24))
+
+
+class QpExUDSendImm(UDResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+
+class QpExRCSendImm(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+
+class QpExXRCSendImm(XRCResources):
+ def create_qp(self):
+ qp_attr = QPAttr(port_num=self.ib_port)
+ qp_attr.pkey_index = 0
+ for _ in range(self.qp_count):
+ attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
+ comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
+ xrcd=self.xrcd)
+ qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+ e.IBV_ACCESS_REMOTE_READ
+ recv_qp = QP(self.ctx, attr_ex, qp_attr)
+ self.rqp_lst.append(recv_qp)
+
+ send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND,
+ e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+ self.sqp_lst.append(send_qp)
+ self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
+ self.psns.append(random.getrandbits(24))
+
+
+class QpExRCRDMAWrite(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_WRITE)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
+
+
+class QpExRCRDMAWriteImm(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC,
+ e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
+
+
+class QpExRCRDMARead(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_READ)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_READ)
+
+
+class QpExRCAtomicCmpSwp(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC,
+ e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_ATOMIC)
+
+
+class QpExRCAtomicFetchAdd(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC,
+ e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_ATOMIC)
+
+
+class QpExRCBindMw(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_BIND_MW)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
+
+
+class QpExTestCase(RDMATestCase):
+ """ Run traffic using the new post send API. """
+ def setUp(self):
+ super().setUp()
+ self.iters = 100
+ self.qp_dict = {'ud_send': QpExUDSend, 'rc_send': QpExRCSend,
+ 'xrc_send': QpExXRCSend, 'ud_send_imm': QpExUDSendImm,
+ 'rc_send_imm': QpExRCSendImm,
+ 'xrc_send_imm': QpExXRCSendImm,
+ 'rc_write': QpExRCRDMAWrite,
+ 'rc_write_imm': QpExRCRDMAWriteImm,
+ 'rc_read': QpExRCRDMARead,
+ 'rc_cmp_swp': QpExRCAtomicCmpSwp,
+ 'rc_fetch_add': QpExRCAtomicFetchAdd,
+ 'rc_bind_mw': QpExRCBindMw}
+
+ def create_players(self, qp_type):
+ client = self.qp_dict[qp_type](self.dev_name, self.ib_port,
+ self.gid_index)
+ server = self.qp_dict[qp_type](self.dev_name, self.ib_port,
+ self.gid_index)
+ if 'xrc' in qp_type:
+ client.pre_run(server.psns, server.qps_num)
+ server.pre_run(client.psns, client.qps_num)
+ else:
+ client.pre_run(server.psn, server.qpn)
+ server.pre_run(client.psn, client.qpn)
+ return client, server
+
+ def test_qp_ex_ud_send(self):
+ client, server = self.create_players('ud_send')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND)
+
+ def test_qp_ex_rc_send(self):
+ client, server = self.create_players('rc_send')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND)
+
+ def test_qp_ex_xrc_send(self):
+ client, server = self.create_players('xrc_send')
+ u.xrc_traffic(client, server, send_op=e.IBV_QP_EX_WITH_SEND)
+
+ def test_qp_ex_ud_send_imm(self):
+ client, server = self.create_players('ud_send_imm')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+ def test_qp_ex_rc_send_imm(self):
+ client, server = self.create_players('rc_send_imm')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+ def test_qp_ex_xrc_send_imm(self):
+ client, server = self.create_players('xrc_send_imm')
+ u.xrc_traffic(client, server, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+ def test_qp_ex_rc_rdma_write(self):
+ client, server = self.create_players('rc_write')
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE)
+
+ def test_qp_ex_rc_rdma_write_imm(self):
+ client, server = self.create_players('rc_write_imm')
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
+
+ def test_qp_ex_rc_rdma_read(self):
+ client, server = self.create_players('rc_read')
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ server.mr.write('s' * server.msg_size, server.msg_size)
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_READ)
+
+ def test_qp_ex_rc_atomic_cmp_swp(self):
+ client, server = self.create_players('rc_cmp_swp')
+ client.msg_size = 8 # Atomic work on 64b operators
+ server.msg_size = 8
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ server.mr.write('s' * 8, 8)
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
+
+ def test_qp_ex_rc_atomic_fetch_add(self):
+ client, server = self.create_players('rc_fetch_add')
+ client.msg_size = 8 # Atomic work on 64b operators
+ server.msg_size = 8
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ server.mr.write('s' * 8, 8)
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
+
+ def test_qp_ex_rc_bind_mw(self):
+ """
+ Verify bind memory window operation using the new post_send API.
+ Instead of checking through regular pingpong style traffic, we'll
+ do as follows:
+ - Register an MR with remote write access
+ - Bind a MW without remote write permission to the MR
+ - Verify that remote write fails
+ Since it's a unique flow, it's an integral part of that test rather
+ than a utility method.
+ """
+ client, server = self.create_players('rc_bind_mw')
+ client_sge = u.get_send_element(client, False)[1]
+ # Create a MW and bind it
+ server.qp.wr_start()
+ server.qp.wr_id = 0x123
+ server.qp.wr_flags = e.IBV_SEND_SIGNALED
+ bind_info = MWBindInfo(server.mr, server.mr.buf, server.mr.length,
+ e.IBV_ACCESS_LOCAL_WRITE)
+ mw = MW(server.pd, mw_type=e.IBV_MW_TYPE_2)
+ new_key = inc_rkey(server.mr.rkey)
+ server.qp.wr_bind_mw(mw, new_key, bind_info)
+ server.qp.wr_complete()
+ u.poll_cq(server.cq)
+ # Verify remote write fails
+ client.qp.wr_start()
+ client.qp.wr_id = 0x124
+ client.qp.wr_flags = e.IBV_SEND_SIGNALED
+ client.qp.wr_rdma_write(new_key, server.mr.buf)
+ client.qp.wr_set_sge(client_sge)
+ client.qp.wr_complete()
+ try:
+ u.poll_cq(client.cq)
+ except PyverbsRDMAError as exp:
+ if 'Completion status is Remote access error' not in exp.args[0]:
+ raise exp
+
diff --git a/tests/utils.py b/tests/utils.py
index 47eacfee35e5..20132a7cf40b 100755
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -7,6 +7,7 @@ from itertools import combinations as com
from string import ascii_lowercase as al
import unittest
import random
+import socket
from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError
from pyverbs.addr import AHAttr, AH, GlobalRoute
@@ -16,6 +17,7 @@ from tests.base import XRCResources
from pyverbs.cq import PollCqAttr
import pyverbs.device as d
import pyverbs.enums as e
+from pyverbs.mr import MR
MAX_MR_SIZE = 4194304
# Some HWs limit DM address and length alignment to 4 for read and write
@@ -29,6 +31,7 @@ MAX_DM_LOG_ALIGN = 6
# Raw Packet QP supports TSO header, which creates a larger send WQE.
MAX_RAW_PACKET_SEND_WR = 2500
GRH_SIZE = 40
+IMM_DATA = 1234
def get_mr_length():
@@ -197,7 +200,7 @@ def random_qp_create_flags(qpt, attr_ex):
def random_qp_init_attr_ex(attr_ex, attr, qpt=None):
"""
- Create a random-valued QPInitAttrEX object with the given QP type.
+ Create a random-valued QPInitAttrEx object with the given QP type.
QP type affects QP capabilities, so allow users to set it and still get
valid attributes.
:param attr_ex: Extended device attributes for capability checks
@@ -251,24 +254,38 @@ def wc_status_to_str(status):
except KeyError:
return 'Unknown WC status ({s})'.format(s=status)
+
+def create_custom_mr(agr_obj, additional_access_flags=0, size=None):
+ """
+ Creates a memory region using the aggregation object's PD.
+ If size is None, the agr_obj's message size is used to set the MR's size.
+ The access flags are local write and the additional_access_flags.
+ :param agr_obj: The aggregation object that creates the MR
+ :param additional_access_flags: Addition access flags to set in the MR
+ :param size: MR's length. If None, agr_obj.msg_size is used.
+ """
+ mr_length = size if size else agr_obj.msg_size
+ return MR(agr_obj.pd, mr_length,
+ e.IBV_ACCESS_LOCAL_WRITE | additional_access_flags)
+
# Traffic helpers
-def get_send_wr(agr_obj, is_server):
+def get_send_element(agr_obj, is_server):
"""
- Creates a single SGE Send WR for agr_obj's QP type. The content of the
- message is either 's' for server side or 'c' for client side.
+ Creates a single SGE and a single Send WR for agr_obj's QP type. The content
+ of the message is either 's' for server side or 'c' for client side.
:param agr_obj: Aggregation object which contains all resources necessary
:param is_server: Indicates whether this is server or client side
- :return: send wr
+ :return: send wr and its SGE
"""
+ mr = agr_obj.mr
qp_type = agr_obj.sqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
else agr_obj.qp.qp_type
- mr = agr_obj.mr
offset = GRH_SIZE if qp_type == e.IBV_QPT_UD else 0
- send_sge = SGE(mr.buf + offset, agr_obj.msg_size, mr.lkey)
msg = (agr_obj.msg_size + offset) * ('s' if is_server else 'c')
mr.write(msg, agr_obj.msg_size + offset)
- return SendWR(num_sge=1, sg=[send_sge])
+ sge = SGE(mr.buf + offset, agr_obj.msg_size, mr.lkey)
+ return SendWR(num_sge=1, sg=[sge]), sge
def get_recv_wr(agr_obj):
@@ -286,6 +303,64 @@ def get_recv_wr(agr_obj):
return RecvWR(sg=[recv_sge], num_sge=1)
+def get_global_ah(agr_obj, gid_index, port):
+ gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
+ sgid_index=gid_index)
+ ah_attr = AHAttr(port_num=port, is_global=1, gr=gr,
+ dlid=agr_obj.port_attr.lid)
+ return AH(agr_obj.pd, attr=ah_attr)
+
+
+def xrc_post_send(agr_obj, qp_num, send_object, gid_index, port, send_op=None):
+ agr_obj.qp = agr_obj.sqp_lst[qp_num]
+ if send_op:
+ post_send_ex(agr_obj, send_object, gid_index, port, send_op)
+ else:
+ post_send(agr_obj, send_object, gid_index, port)
+
+
+def post_send_ex(agr_obj, send_object, gid_index, port, send_op=None):
+ qp_type = agr_obj.qp.qp_type
+ agr_obj.qp.wr_start()
+ agr_obj.qp.wr_id = 0x123
+ agr_obj.qp.wr_flags = e.IBV_SEND_SIGNALED
+ if send_op == e.IBV_QP_EX_WITH_SEND:
+ agr_obj.qp.wr_send()
+ elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE:
+ agr_obj.qp.wr_rdma_write(agr_obj.rkey, agr_obj.raddr)
+ elif send_op == e.IBV_QP_EX_WITH_SEND_WITH_IMM:
+ agr_obj.qp.wr_send_imm(IMM_DATA)
+ elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM:
+ agr_obj.qp.wr_rdma_write_imm(agr_obj.rkey, agr_obj.raddr, IMM_DATA)
+ elif send_op == e.IBV_QP_EX_WITH_RDMA_READ:
+ agr_obj.qp.wr_rdma_read(agr_obj.rkey, agr_obj.raddr)
+ elif send_op == e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP:
+ # We're checking the returned value (remote's content), so cmp/swp
+ # values are of no importance.
+ agr_obj.qp.wr_atomic_cmp_swp(agr_obj.rkey, agr_obj.raddr, 42, 43)
+ elif send_op == e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD:
+ agr_obj.qp.wr_atomic_fetch_add(agr_obj.rkey, agr_obj.raddr, 1)
+ elif send_op == e.IBV_QP_EX_WITH_BIND_MW:
+ bind_info = MWBindInfo(agr_obj.mr, agr_obj.mr.buf, agr_obj.mr.rkey,
+ e.IBV_ACCESS_REMOTE_WRITE)
+ mw = MW(agr_obj.pd, mw_type=e.IBV_MW_TYPE_2)
+ # A new rkey is needed to be set into bind_info, modify rkey
+ agr_obj.qp.wr_bind_mw(mw, agr_obj.mr.rkey + 12, bind_info)
+ agr_obj.qp.wr_complete()
+ return
+ #agr_obj.qp.wr_start()
+ #agr_obj.qp.wr_id = 0x123
+ #agr_obj.qp.wr_flags = e.IBV_SEND_SIGNALED
+ #agr_obj.qp.wr_send()
+ if qp_type == e.IBV_QPT_UD:
+ ah = get_global_ah(agr_obj, gid_index, port)
+ agr_obj.qp.wr_set_ud_addr(ah, agr_obj.rqpn, agr_obj.UD_QKEY)
+ if qp_type == e.IBV_QPT_XRC_SEND:
+ agr_obj.qp.wr_set_xrc_srqn(agr_obj.remote_srqn)
+ agr_obj.qp.wr_set_sge(send_object)
+ agr_obj.qp.wr_complete()
+
+
def post_send(agr_obj, send_wr, gid_index, port):
"""
Post a single send WR to the QP. Post_send's second parameter (send bad wr)
@@ -299,11 +374,7 @@ def post_send(agr_obj, send_wr, gid_index, port):
"""
qp_type = agr_obj.qp.qp_type
if qp_type == e.IBV_QPT_UD:
- gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
- sgid_index=gid_index)
- ah_attr = AHAttr(port_num=port, is_global=1, gr=gr,
- dlid=agr_obj.port_attr.lid)
- ah = AH(agr_obj.pd, attr=ah_attr)
+ ah = get_global_ah(agr_obj, gid_index, port)
send_wr.set_wr_ud(ah, agr_obj.rqpn, agr_obj.UD_QKEY)
agr_obj.qp.post_send(send_wr, None)
@@ -321,7 +392,7 @@ def post_recv(qp, recv_wr, num_wqes=1):
qp.post_recv(recv_wr, None)
-def poll_cq(cq, count=1):
+def poll_cq(cq, count=1, data=None):
"""
Poll <count> completions from the CQ.
Note: This function calls the blocking poll() method of the CQ
@@ -329,6 +400,8 @@ def poll_cq(cq, count=1):
single CQ event when events are used.
:param cq: CQ to poll from
:param count: How many completions to poll
+ :param data: In case of a work request with immediate, the immediate data
+ to be compared after poll
:return: An array of work completions of length <count>, None
when events are used
"""
@@ -339,15 +412,21 @@ def poll_cq(cq, count=1):
if wc.status != e.IBV_WC_SUCCESS:
raise PyverbsRDMAError('Completion status is {s}'.
format(s=wc_status_to_str(wc.status)))
+ if data:
+ if wc.wc_flags & e.IBV_WC_WITH_IMM == 0:
+ raise PyverbsRDMAError('Completion without immediate')
+ assert socket.ntohl(wc.imm_data) == data
count -= nc
return wcs
-def poll_cq_ex(cqex, count=1):
+def poll_cq_ex(cqex, count=1, data=None):
"""
Poll <count> completions from the extended CQ.
:param cq: CQEX to poll from
:param count: How many completions to poll
+ :param data: In case of a work request with immediate, the immediate data
+ to be compared after poll
:return: None
"""
poll_attr = PollCqAttr()
@@ -360,6 +439,8 @@ def poll_cq_ex(cqex, count=1):
if cqex.status != e.IBV_WC_SUCCESS:
raise PyverbsRDMAErrno('Completion status is {s}'.
format(s=cqex.status))
+ if data:
+ assert data == socket.ntohl(cqex.read_imm_data())
# Now poll the rest of the packets
while count > 0:
ret = cqex.poll_next()
@@ -370,6 +451,8 @@ def poll_cq_ex(cqex, count=1):
if cqex.status != e.IBV_WC_SUCCESS:
raise PyverbsRDMAErrno('Completion status is {s}'.
format(s=cqex.status))
+ if data:
+ assert data == socket.ntohl(cqex.read_imm_data())
count -= 1
cqex.end_poll()
@@ -398,7 +481,13 @@ def validate(received_str, is_server, msg_size):
format(exp=expected_str, rcv=received_str))
-def traffic(client, server, iters, gid_idx, port, is_cq_ex=False):
+def send(agr_obj, send_wr, gid_index, port, send_op=None):
+ if send_op:
+ return post_send_ex(agr_obj, send_wr, gid_index, port, send_op)
+ return post_send(agr_obj, send_wr, gid_index, port)
+
+
+def traffic(client, server, iters, gid_idx, port, is_cq_ex=False, send_op=None):
"""
Runs basic traffic between two sides
:param client: client side, clients base class is BaseTraffic
@@ -407,32 +496,83 @@ def traffic(client, server, iters, gid_idx, port, is_cq_ex=False):
:param gid_idx: local gid index
:param port: IB port
:param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
+ :param send_op: If not None, new post send API is assumed.
:return:
"""
poll = poll_cq_ex if is_cq_ex else poll_cq
+ if send_op == e.IBV_QP_EX_WITH_SEND_WITH_IMM or \
+ send_op == e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM:
+ imm_data = IMM_DATA
+ else:
+ imm_data = None
+ # Using the new post send API, we need the SGE, not the SendWR
+ send_element_idx = 1 if send_op else 0
s_recv_wr = get_recv_wr(server)
c_recv_wr = get_recv_wr(client)
post_recv(client.qp, c_recv_wr, client.num_msgs)
post_recv(server.qp, s_recv_wr, server.num_msgs)
read_offset = GRH_SIZE if client.qp.qp_type == e.IBV_QPT_UD else 0
for _ in range(iters):
- c_send_wr = get_send_wr(client, False)
- post_send(client, c_send_wr, gid_idx, port)
+ c_send_wr = get_send_element(client, False)[send_element_idx]
+ send(client, c_send_wr, gid_idx, port, send_op)
poll(client.cq)
- poll(server.cq)
+ poll(server.cq, data=imm_data)
post_recv(server.qp, s_recv_wr)
msg_received = server.mr.read(server.msg_size, read_offset)
validate(msg_received, True, server.msg_size)
- s_send_wr = get_send_wr(server, True)
- post_send(server, s_send_wr, gid_idx, port)
+ s_send_wr = get_send_element(server, True)[send_element_idx]
+ send(server, s_send_wr, gid_idx, port, send_op)
poll(server.cq)
- poll(client.cq)
+ poll(client.cq, data=imm_data)
post_recv(client.qp, c_recv_wr)
msg_received = client.mr.read(client.msg_size, read_offset)
validate(msg_received, False, client.msg_size)
-def xrc_traffic(client, server, is_cq_ex=False):
+def rdma_traffic(client, server, iters, gid_idx, port, is_cq_ex=False, send_op=None):
+ """
+ Runs basic RDMA traffic between two sides. No receive WQEs are posted. For
+ RDMA send with immediate, use traffic().
+ :param client: client side, clients base class is BaseTraffic
+ :param server: server side, servers base class is BaseTraffic
+ :param iters: number of traffic iterations
+ :param gid_idx: local gid index
+ :param port: IB port
+ :param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
+ :param send_op: If not None, new post send API is assumed.
+ :return:
+ """
+ # Using the new post send API, we need the SGE, not the SendWR
+ send_element_idx = 1 if send_op else 0
+ same_side_check = (send_op == e.IBV_QP_EX_WITH_RDMA_READ or
+ send_op == e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP or
+ send_op == e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
+ for _ in range(iters):
+ c_send_wr = get_send_element(client, False)[send_element_idx]
+ send(client, c_send_wr, gid_idx, port, send_op)
+ poll_cq(client.cq)
+ if same_side_check:
+ msg_received = client.mr.read(client.msg_size, 0)
+ else:
+ msg_received = server.mr.read(server.msg_size, 0)
+ validate(msg_received, False if same_side_check else True,
+ server.msg_size)
+ s_send_wr = get_send_element(server, True)[send_element_idx]
+ if same_side_check:
+ client.mr.write('c' * client.msg_size, client.msg_size)
+ send(server, s_send_wr, gid_idx, port, send_op)
+ poll_cq(server.cq)
+ if same_side_check:
+ msg_received = server.mr.read(client.msg_size, 0)
+ else:
+ msg_received = client.mr.read(server.msg_size, 0)
+ validate(msg_received, True if same_side_check else False,
+ client.msg_size)
+ if same_side_check:
+ server.mr.write('s' * server.msg_size, server.msg_size)
+
+
+def xrc_traffic(client, server, is_cq_ex=False, send_op=None):
"""
Runs basic xrc traffic, this function assumes that number of QPs, which
server and client have are equal, server.send_qp[i] is connected to
@@ -444,27 +584,32 @@ def xrc_traffic(client, server, is_cq_ex=False):
:param server: Aggregation object of the passive side, should be an instance
of XRCResources class
:param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
+ :param send_op: If not None, new post send API is assumed.
:return: None
"""
poll = poll_cq_ex if is_cq_ex else poll_cq
- client_srqn = client.srq.get_srq_num()
- server_srqn = server.srq.get_srq_num()
+ server.remote_srqn = client.srq.get_srq_num()
+ client.remote_srqn = server.srq.get_srq_num()
s_recv_wr = get_recv_wr(server)
c_recv_wr = get_recv_wr(client)
post_recv(client.srq, c_recv_wr, client.qp_count*client.num_msgs)
post_recv(server.srq, s_recv_wr, server.qp_count*server.num_msgs)
+ # Using the new post send API, we need the SGE, not the SendWR
+ send_element_idx = 1 if send_op else 0
for _ in range(client.num_msgs):
for i in range(server.qp_count):
- c_send_wr = get_send_wr(client, False)
- c_send_wr.set_qp_type_xrc(server_srqn)
- client.sqp_lst[i].post_send(c_send_wr)
+ c_send_wr = get_send_element(client, False)[send_element_idx]
+ if send_op is None:
+ c_send_wr.set_qp_type_xrc(client.remote_srqn)
+ xrc_post_send(client, i, c_send_wr, 0, 0, send_op)
poll(client.cq)
poll(server.cq)
msg_received = server.mr.read(server.msg_size, 0)
validate(msg_received, True, server.msg_size)
- s_send_wr = get_send_wr(server, True)
- s_send_wr.set_qp_type_xrc(client_srqn)
- server.sqp_lst[i].post_send(s_send_wr)
+ s_send_wr = get_send_element(server, True)[send_element_idx]
+ if send_op is None:
+ s_send_wr.set_qp_type_xrc(server.remote_srqn)
+ xrc_post_send(server, i, s_send_wr, 0, 0, send_op)
poll(server.cq)
poll(client.cq)
msg_received = client.mr.read(client.msg_size, 0)
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
end of thread, other threads:[~2019-12-31 9:20 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 2/8] pyverbs: Add TSO support Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 4/8] tests: Use post_recv in the right place Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 5/8] pyverbs: Expose MR's length property Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 8/8] tests: Add test using the new post send API Noa Osherovich
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.