* [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API
@ 2019-12-31 9:19 Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
` (7 more replies)
0 siblings, 8 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Noa Osherovich
The following series adds pyverbs support for the new post send API.
The first 5 patches add unrelated but needed support (e.g. memory
window) and fixes for bugs that were found along the way.
These patches are followed by the feature itself, a documentation and
a test.
Noa Osherovich (8):
pyverbs: Add support for memory window
pyverbs: Add TSO support
tests: Decrease maximal TSO header size
tests: Use post_recv in the right place
pyverbs: Expose MR's length property
pyverbs: Introduce extended QP and new post send
Documentation: Add extended QP to pyverbs's doc
tests: Add test using the new post send API
Documentation/pyverbs.md | 24 +++
pyverbs/base.pyx | 8 +-
pyverbs/libibverbs.pxd | 48 +++++-
pyverbs/libibverbs_enums.pxd | 14 ++
pyverbs/mr.pxd | 6 +
pyverbs/mr.pyx | 23 +++
pyverbs/qp.pxd | 6 +
pyverbs/qp.pyx | 154 +++++++++++++++++-
pyverbs/wr.pxd | 6 +
pyverbs/wr.pyx | 60 ++++++-
tests/CMakeLists.txt | 1 +
tests/test_odp.py | 13 +-
tests/test_qpex.py | 295 +++++++++++++++++++++++++++++++++++
tests/utils.py | 213 +++++++++++++++++++++----
14 files changed, 817 insertions(+), 54 deletions(-)
mode change 100755 => 100644 tests/CMakeLists.txt
create mode 100644 tests/test_qpex.py
--
2.21.0
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH rdma-core 1/8] pyverbs: Add support for memory window
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 2/8] pyverbs: Add TSO support Noa Osherovich
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Ahmad Ghazawi, Edward Srouji
In order to use memory window, some additions were needed:
- Allow pyverbs users to create MWBindInfo, the Python representation
of ibv_mw_bind_info.
- Expose ibv_inc_rkey, which creates a new rkey from a given one by
increasing its 8 LSBs while keeping the same index.
- Expose the memory window's rkey and handle properties.
Signed-off-by: Ahmad Ghazawi <ahmadg@mellanox.com>
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/base.pyx | 8 +++++++-
pyverbs/libibverbs.pxd | 1 +
pyverbs/mr.pxd | 6 ++++++
pyverbs/mr.pyx | 19 +++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx
index c5b16795ddb6..790ba4153dea 100644
--- a/pyverbs/base.pyx
+++ b/pyverbs/base.pyx
@@ -1,9 +1,15 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from libc.errno cimport errno
import logging
+
from pyverbs.pyverbs_error import PyverbsRDMAError
-from libc.errno cimport errno
+cimport pyverbs.libibverbs as v
+
+
+def inc_rkey(rkey):
+ return v.ibv_inc_rkey(rkey)
cpdef PyverbsRDMAErrno(str msg):
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index ad8d8bacc541..fea8a1b408da 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -551,3 +551,4 @@ cdef extern from 'infiniband/verbs.h':
ibv_recv_wr **bad_recv_wr)
ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
ibv_parent_domain_init_attr *attr)
+ uint32_t ibv_inc_rkey(uint32_t rkey)
diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd
index fb46611e6f42..402df4492425 100644
--- a/pyverbs/mr.pxd
+++ b/pyverbs/mr.pxd
@@ -12,6 +12,12 @@ cdef class MR(PyverbsCM):
cdef v.ibv_mr *mr
cdef void *buf
cpdef read(self, length, offset)
+ cdef add_ref(self, obj)
+ cdef object bind_infos
+
+cdef class MWBindInfo(PyverbsCM):
+ cdef v.ibv_mw_bind_info info
+ cdef object mr
cdef class MW(PyverbsCM):
cdef object pd
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 6b28c8173ef8..9b1277f0882f 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -3,10 +3,12 @@
import resource
import logging
+import weakref
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
from posix.stdlib cimport posix_memalign
+from pyverbs.base cimport close_weakrefs
from libc.string cimport memcpy, memset
from libc.stdint cimport uintptr_t
from pyverbs.device cimport DM
@@ -29,6 +31,7 @@ cdef class MR(PyverbsCM):
:return: The newly created MR on success
"""
super().__init__()
+ self.bind_infos = weakref.WeakSet()
if self.mr != NULL:
return
#We want to enable registering an MR of size 0 but this fails with a
@@ -61,6 +64,7 @@ cdef class MR(PyverbsCM):
:return: None
"""
self.logger.debug('Closing MR')
+ close_weakrefs([self.bind_infos])
if self.mr != NULL:
rc = v.ibv_dereg_mr(self.mr)
if rc != 0:
@@ -96,6 +100,10 @@ cdef class MR(PyverbsCM):
data = <char*>(self.buf + off)
return data[:length]
+ cdef add_ref(self, obj):
+ if isinstance(obj, MWBindInfo):
+ self.bind_infos.add(obj)
+
@property
def buf(self):
return <uintptr_t>self.buf
@@ -109,6 +117,17 @@ cdef class MR(PyverbsCM):
return self.mr.rkey
+cdef class MWBindInfo(PyverbsCM):
+ def __init__(self, MR mr not None, addr, length, mw_access_flags):
+ super().__init__()
+ self.mr = mr
+ self.info.mr = mr.mr
+ self.info.addr = addr
+ self.info.length = length
+ self.info.mw_access_flags = mw_access_flags
+ mr.add_ref(self)
+
+
cdef class MW(PyverbsCM):
def __init__(self, PD pd not None, v.ibv_mw_type mw_type):
"""
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 2/8] pyverbs: Add TSO support
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size Noa Osherovich
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Ahmad Ghazawi, Edward Srouji
Add TSO class to allow users to easily set the TSO field in a send
work request.
Signed-off-by: Ahmad Ghazawi <ahmadg@mellanox.com>
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/libibverbs.pxd | 7 ++---
pyverbs/wr.pxd | 6 +++++
pyverbs/wr.pyx | 60 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 66 insertions(+), 7 deletions(-)
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index fea8a1b408da..8949759511a5 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -311,10 +311,6 @@ cdef extern from 'infiniband/verbs.h':
unsigned short hdr_sz
unsigned short mss
- cdef union unnamed:
- bind_mw bind_mw
- tso tso
-
cdef struct xrc:
unsigned int remote_srqn
@@ -330,7 +326,8 @@ cdef extern from 'infiniband/verbs.h':
unsigned int send_flags
wr wr
qp_type qp_type
- unnamed unnamed
+ bind_mw bind_mw
+ tso tso
cdef struct ibv_qp_cap:
unsigned int max_send_wr
diff --git a/pyverbs/wr.pxd b/pyverbs/wr.pxd
index e259249ef7f8..5cb282dc65c3 100644
--- a/pyverbs/wr.pxd
+++ b/pyverbs/wr.pxd
@@ -16,3 +16,9 @@ cdef class RecvWR(PyverbsCM):
cdef class SendWR(PyverbsCM):
cdef v.ibv_send_wr send_wr
+
+cdef class TSO(PyverbsCM):
+ cdef void* buf
+ cdef int length
+ cdef int mss
+ cpdef alloc_buf(self, content, length)
diff --git a/pyverbs/wr.pyx b/pyverbs/wr.pyx
index 06186e9b0fd2..b1df3677fce3 100644
--- a/pyverbs/wr.pyx
+++ b/pyverbs/wr.pyx
@@ -1,12 +1,16 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies Inc. All rights reserved. See COPYING file
+import resource
+
+from cpython.pycapsule cimport PyCapsule_New
+from libc.stdlib cimport free, malloc
+from libc.string cimport memcpy
+
from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
from pyverbs.addr cimport AH
-from libc.stdlib cimport free, malloc
-from libc.string cimport memcpy
cdef class SGE(PyverbsCM):
@@ -281,6 +285,58 @@ cdef class SendWR(PyverbsCM):
"""
self.send_wr.qp_type.xrc.remote_srqn = remote_srqn
+ def set_tso(self, TSO tso not None):
+ """
+ Set the members of the tso struct in the send_wr's anonymous union.
+ :param tso: A TSO object to copy to the work request
+ :return: None
+ """
+ self.send_wr.tso.hdr = tso.buf
+ self.send_wr.tso.hdr_sz = tso.length
+ self.send_wr.tso.mss = tso.mss
+
+
+cdef class TSO(PyverbsCM):
+ """ Represents the 'tso' anonymous struct inside a send WR """
+ def __init__(self, data, length, mss):
+ super().__init__()
+ self.alloc_buf(data, length)
+ self.length = length
+ self.mss = mss
+
+ cpdef alloc_buf(self, data, length):
+ if self.buf != NULL:
+ free(self.buf)
+ self.buf = malloc(length)
+ if self.buf == NULL:
+ raise PyverbsError('Failed to allocate TSO buffer of length {l}'.\
+ format(l=length))
+ if isinstance(data, str):
+ data = data.encode()
+ memcpy(self.buf, <char*>data, length)
+ self.length = length
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ if self.buf != NULL:
+ free(self.buf)
+ self.buf = NULL
+
+ @property
+ def buf(self):
+ return PyCapsule_New(self.buf, NULL, NULL)
+
+ def __str__(self):
+ print_format = '{:22}: {:<20}\n'
+ data = <char*>(self.buf)
+ data = data[:self.length].decode()
+ return print_format.format('MSS', self.mss) +\
+ print_format.format('Length', self.length) +\
+ print_format.format('Data', data)
+
+
def send_flags_to_str(flags):
send_flags = {e.IBV_SEND_FENCE: 'IBV_SEND_FENCE',
e.IBV_SEND_SIGNALED: 'IBV_SEND_SIGNALED',
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 2/8] pyverbs: Add TSO support Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 4/8] tests: Use post_recv in the right place Noa Osherovich
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
When creating a Raw Packet QP with a TSO header, send WQE size
increases to accommodate it. If both max_tso_header and max_send_wr
are too large, the requested WQE size can become too large for the
device to support.
Decrease max_tso_header's max value to avoid that.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
tests/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/utils.py b/tests/utils.py
index c45170dbd329..d4d0d1ef49ef 100755
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -224,7 +224,7 @@ def random_qp_init_attr_ex(attr_ex, attr, qpt=None):
mask -= e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER
else:
max_tso = \
- random.randint(16, int(attr_ex.tso_caps.max_tso / 400))
+ random.randint(16, int(attr_ex.tso_caps.max_tso / 800))
qia = QPInitAttrEx(qp_type=qpt, cap=qp_cap, sq_sig_all=sig, comp_mask=mask,
create_flags=cflags, max_tso_header=max_tso)
if mask & e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER:
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 4/8] tests: Use post_recv in the right place
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (2 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 5/8] pyverbs: Expose MR's length property Noa Osherovich
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
In traffic() helper method, do post_recv after a recv WQE was consumed
by the hardware.
Fixes: 6fb2b9bade55 ('tests: Add traffic helper methods')
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
tests/utils.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/utils.py b/tests/utils.py
index d4d0d1ef49ef..47eacfee35e5 100755
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -420,14 +420,14 @@ def traffic(client, server, iters, gid_idx, port, is_cq_ex=False):
post_send(client, c_send_wr, gid_idx, port)
poll(client.cq)
poll(server.cq)
- post_recv(client.qp, c_recv_wr)
+ post_recv(server.qp, s_recv_wr)
msg_received = server.mr.read(server.msg_size, read_offset)
validate(msg_received, True, server.msg_size)
s_send_wr = get_send_wr(server, True)
post_send(server, s_send_wr, gid_idx, port)
poll(server.cq)
poll(client.cq)
- post_recv(server.qp, s_recv_wr)
+ post_recv(client.qp, c_recv_wr)
msg_received = client.mr.read(client.msg_size, read_offset)
validate(msg_received, False, client.msg_size)
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 5/8] pyverbs: Expose MR's length property
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (3 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 4/8] tests: Use post_recv in the right place Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send Noa Osherovich
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
MR's length is needed for testing purposes.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/mr.pyx | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 9b1277f0882f..234e864209a2 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -116,6 +116,10 @@ cdef class MR(PyverbsCM):
def rkey(self):
return self.mr.rkey
+ @property
+ def length(self):
+ return self.mr.length
+
cdef class MWBindInfo(PyverbsCM):
def __init__(self, MR mr not None, addr, length, mw_access_flags):
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (4 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 5/8] pyverbs: Expose MR's length property Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 8/8] tests: Add test using the new post send API Noa Osherovich
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Ahmad Ghazawi, Edward Srouji
Add QPEx class alongside the new post send functions.
The new post send flow is as follows:
- Create a WR on the extended QP.
- Set the WR's attributes.
- Commit the entries in the send buffer and ring the doorbell.
QPEx class exposes the relevant methods (wr_*) for that.
Signed-off-by: Ahmad Ghazawi <ahmadg@mellanox.com>
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
pyverbs/libibverbs.pxd | 40 +++++++++
pyverbs/libibverbs_enums.pxd | 14 ++++
pyverbs/qp.pxd | 6 ++
pyverbs/qp.pyx | 154 +++++++++++++++++++++++++++++++++--
4 files changed, 209 insertions(+), 5 deletions(-)
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index 8949759511a5..c038c4aa4d5b 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -438,6 +438,10 @@ cdef extern from 'infiniband/verbs.h':
unsigned int handle
unsigned int events_completed
+ cdef struct ibv_data_buf:
+ void *addr
+ size_t length
+
cdef struct ibv_qp:
ibv_context *context;
void *qp_context;
@@ -460,6 +464,12 @@ cdef extern from 'infiniband/verbs.h':
uint64_t resource_type);
void *pd_context;
+ cdef struct ibv_qp_ex:
+ ibv_qp qp_base
+ uint64_t comp_mask
+ uint64_t wr_id
+ unsigned int wr_flags
+
ibv_device **ibv_get_device_list(int *n)
void ibv_free_device_list(ibv_device **list)
ibv_context *ibv_open_device(ibv_device *device)
@@ -549,3 +559,33 @@ cdef extern from 'infiniband/verbs.h':
ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
ibv_parent_domain_init_attr *attr)
uint32_t ibv_inc_rkey(uint32_t rkey)
+ ibv_qp_ex *ibv_qp_to_qp_ex(ibv_qp *qp)
+ void ibv_wr_atomic_cmp_swp(ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t compare,
+ uint64_t swap)
+ void ibv_wr_atomic_fetch_add(ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t add)
+ void ibv_wr_bind_mw(ibv_qp_ex *qp, ibv_mw *mw, uint32_t rkey,
+ ibv_mw_bind_info *bind_info)
+ void ibv_wr_local_inv(ibv_qp_ex *qp, uint32_t invalidate_rkey)
+ void ibv_wr_rdma_read(ibv_qp_ex *qp, uint32_t rkey, uint64_t remote_addr)
+ void ibv_wr_rdma_write(ibv_qp_ex *qp, uint32_t rkey, uint64_t remote_addr)
+ void ibv_wr_rdma_write_imm(ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint32_t imm_data)
+ void ibv_wr_send(ibv_qp_ex *qp)
+ void ibv_wr_send_imm(ibv_qp_ex *qp, uint32_t imm_data)
+ void ibv_wr_send_inv(ibv_qp_ex *qp, uint32_t invalidate_rkey)
+ void ibv_wr_send_tso(ibv_qp_ex *qp, void *hdr, uint16_t hdr_sz,
+ uint16_t mss)
+ void ibv_wr_set_ud_addr(ibv_qp_ex *qp, ibv_ah *ah, uint32_t remote_qpn,
+ uint32_t remote_qkey)
+ void ibv_wr_set_xrc_srqn(ibv_qp_ex *qp, uint32_t remote_srqn)
+ void ibv_wr_set_inline_data(ibv_qp_ex *qp, void *addr, size_t length)
+ void ibv_wr_set_inline_data_list(ibv_qp_ex *qp, size_t num_buf,
+ ibv_data_buf *buf_list)
+ void ibv_wr_set_sge(ibv_qp_ex *qp, uint32_t lkey, uint64_t addr,
+ uint32_t length)
+ void ibv_wr_set_sge_list(ibv_qp_ex *qp, size_t num_sge, ibv_sge *sg_list)
+ void ibv_wr_start(ibv_qp_ex *qp)
+ int ibv_wr_complete(ibv_qp_ex *qp)
+ void ibv_wr_abort(ibv_qp_ex *qp)
diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd
index fd6a6f49a163..a706cec0aba8 100755
--- a/pyverbs/libibverbs_enums.pxd
+++ b/pyverbs/libibverbs_enums.pxd
@@ -236,6 +236,7 @@ cdef extern from '<infiniband/verbs.h>':
IBV_QP_INIT_ATTR_MAX_TSO_HEADER
IBV_QP_INIT_ATTR_IND_TABLE
IBV_QP_INIT_ATTR_RX_HASH
+ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
cpdef enum ibv_qp_create_flags:
IBV_QP_CREATE_BLOCK_SELF_MCAST_LB
@@ -401,6 +402,19 @@ cdef extern from '<infiniband/verbs.h>':
cpdef enum:
IBV_WC_STANDARD_FLAGS
+ cpdef enum ibv_qp_create_send_ops_flags:
+ IBV_QP_EX_WITH_RDMA_WRITE
+ IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM
+ IBV_QP_EX_WITH_SEND
+ IBV_QP_EX_WITH_SEND_WITH_IMM
+ IBV_QP_EX_WITH_RDMA_READ
+ IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP
+ IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD
+ IBV_QP_EX_WITH_LOCAL_INV
+ IBV_QP_EX_WITH_BIND_MW
+ IBV_QP_EX_WITH_SEND_WITH_INV
+ IBV_QP_EX_WITH_TSO
+
cdef unsigned long long IBV_DEVICE_RAW_SCATTER_FCS
cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING
diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd
index 52aab503e40d..209a2438dd83 100644
--- a/pyverbs/qp.pxd
+++ b/pyverbs/qp.pxd
@@ -37,3 +37,9 @@ cdef class QP(PyverbsCM):
cdef update_cqs(self, init_attr)
cdef object scq
cdef object rcq
+
+cdef class DataBuffer(PyverbsCM):
+ cdef v.ibv_data_buf data
+
+cdef class QPEx(QP):
+ cdef v.ibv_qp_ex *qp_ex
diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx
index 9d368b62022d..1fcb23909758 100755
--- a/pyverbs/qp.pyx
+++ b/pyverbs/qp.pyx
@@ -1,20 +1,30 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memcpy
+
from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
from pyverbs.utils import access_flags_to_str, mig_state_to_str
-from pyverbs.pyverbs_error import PyverbsUserError
+from pyverbs.wr cimport RecvWR, SendWR, SGE
from pyverbs.base import PyverbsRDMAErrno
-from pyverbs.wr cimport RecvWR, SendWR
+from pyverbs.addr cimport AHAttr, GID, AH
+from pyverbs.mr cimport MW, MWBindInfo
cimport pyverbs.libibverbs_enums as e
-from pyverbs.addr cimport AHAttr, GID
from pyverbs.addr cimport GlobalRoute
from pyverbs.device cimport Context
+from cpython.ref cimport PyObject
from pyverbs.cq cimport CQ, CQEX
cimport pyverbs.libibverbs as v
from pyverbs.xrcd cimport XRCD
from pyverbs.srq cimport SRQ
from pyverbs.pd cimport PD
-from libc.string cimport memcpy
+
+cdef extern from 'Python.h':
+ void* PyLong_AsVoidPtr(object)
+cdef extern from 'endian.h':
+ unsigned long htobe32(unsigned long host_32bits)
cdef class QPCap(PyverbsObject):
@@ -240,7 +250,7 @@ cdef class QPInitAttrEx(PyverbsObject):
SRQ srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0,
PD pd=None, XRCD xrcd=None, create_flags=0,
max_tso_header=0, source_qpn=0, object hash_conf=None,
- object ind_table=None):
+ object ind_table=None, send_ops_flags=0):
"""
Initialize a QPInitAttrEx object with user-defined or default values.
:param qp_type: QP type to be created
@@ -261,6 +271,8 @@ cdef class QPInitAttrEx(PyverbsObject):
set in create_flags)
:param hash_conf: Not yet supported
:param ind_table: Not yet supported
+ :param send_ops_flags: Send opcodes to be supported by the extended QP.
+ Use ibv_qp_create_send_ops_flags enum
:return: An initialized QPInitAttrEx object
"""
super().__init__()
@@ -302,6 +314,7 @@ cdef class QPInitAttrEx(PyverbsObject):
self.attr.create_flags = create_flags
self.attr.max_tso_header = max_tso_header
self.attr.source_qpn = source_qpn
+ self.attr.send_ops_flags = send_ops_flags
@property
def send_cq(self):
@@ -1141,6 +1154,137 @@ cdef class QP(PyverbsCM):
print_format.format(' state', qp_state_to_str(self.qp_state))
+cdef class DataBuffer(PyverbsCM):
+ def __init__(self, addr, length):
+ super().__init__()
+ self.data.addr = PyLong_AsVoidPtr(addr)
+ self.data.length = length
+
+
+cdef class QPEx(QP):
+ def __init__(self, object creator not None, object init_attr not None,
+ QPAttr qp_attr=None):
+ """
+ Initializes a QPEx object. Since this is an extension of a QP, QP
+ creation is done in the parent class. The extended QP is retrieved by
+ casting the ibv_qp to ibv_qp_ex.
+ :return: An initialized QPEx object
+ """
+ super().__init__(creator, init_attr, qp_attr)
+ self.qp_ex = v.ibv_qp_to_qp_ex(self.qp)
+ if self.qp_ex == NULL:
+ raise PyverbsRDMAErrno('Failed to create extended QP')
+
+ @property
+ def comp_mask(self):
+ return self.qp_ex.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.qp_ex.comp_mask = val
+
+ @property
+ def wr_id(self):
+ return self.qp_ex.wr_id
+ @wr_id.setter
+ def wr_id(self, val):
+ self.qp_ex.wr_id = val
+
+ @property
+ def wr_flags(self):
+ return self.qp_ex.wr_flags
+ @wr_flags.setter
+ def wr_flags(self, val):
+ self.qp_ex.wr_flags = val
+
+ def wr_atomic_cmp_swp(self, rkey, remote_addr, compare, swap):
+ v.ibv_wr_atomic_cmp_swp(self.qp_ex, rkey, remote_addr, compare, swap)
+
+ def wr_atomic_fetch_add(self, rkey, remote_addr, add):
+ v.ibv_wr_atomic_fetch_add(self.qp_ex, rkey, remote_addr, add)
+
+ def wr_bind_mw(self, MW mw, rkey, MWBindInfo bind_info):
+ cdef v.ibv_mw_bind_info *info
+ info = &bind_info.info
+ v.ibv_wr_bind_mw(self.qp_ex, <v.ibv_mw*>mw.mw, rkey,
+ <v.ibv_mw_bind_info*>info)
+
+ def wr_local_inv(self, invalidate_rkey):
+ v.ibv_wr_local_inv(self.qp_ex, invalidate_rkey)
+
+ def wr_rdma_read(self, rkey, remote_addr):
+ v.ibv_wr_rdma_read(self.qp_ex, rkey, remote_addr)
+
+ def wr_rdma_write(self, rkey, remote_addr):
+ v.ibv_wr_rdma_write(self.qp_ex, rkey, remote_addr)
+
+ def wr_rdma_write_imm(self, rkey, remote_addr, data):
+ cdef unsigned int imm_data = htobe32(data)
+ v.ibv_wr_rdma_write_imm(self.qp_ex, rkey, remote_addr, imm_data)
+
+ def wr_send(self):
+ v.ibv_wr_send(self.qp_ex)
+
+ def wr_send_imm(self, data):
+ cdef unsigned int imm_data = htobe32(data)
+ return v.ibv_wr_send_imm(self.qp_ex, imm_data)
+
+ def wr_send_inv(self, invalidate_rkey):
+ v.ibv_wr_send_inv(self.qp_ex, invalidate_rkey)
+
+ def wr_send_tso(self, hdr, hdr_sz, mss):
+ ptr = PyLong_AsVoidPtr(hdr)
+ v.ibv_wr_send_tso(self.qp_ex, ptr, hdr_sz, mss)
+
+ def wr_set_ud_addr(self, AH ah, remote_qpn, remote_rkey):
+ v.ibv_wr_set_ud_addr(self.qp_ex, ah.ah, remote_qpn, remote_rkey)
+
+ def wr_set_xrc_srqn(self, remote_srqn):
+ v.ibv_wr_set_xrc_srqn(self.qp_ex, remote_srqn)
+
+ def wr_set_inline_data(self, addr, length):
+ ptr = PyLong_AsVoidPtr(addr)
+ v.ibv_wr_set_inline_data(self.qp_ex, ptr, length)
+
+ def wr_set_inline_data_list(self, num_buf, buf_list):
+ cdef v.ibv_data_buf *data = NULL
+ data = <v.ibv_data_buf*>malloc(num_buf * sizeof(v.ibv_data_buf))
+ if data == NULL:
+ raise PyverbsError('Failed to allocate data buffer')
+ for i in range(num_buf):
+ data_buf = <DataBuffer>buf_list[i]
+ data[i].addr = data_buf.data.addr
+ data[i].length = data_buf.data.length
+ v.ibv_wr_set_inline_data_list(self.qp_ex, num_buf, data)
+ free(data)
+
+ def wr_set_sge(self, SGE sge not None):
+ v.ibv_wr_set_sge(self.qp_ex, sge.lkey, sge.addr, sge.length)
+
+ def wr_set_sge_list(self, num_sge, sg_list):
+ cdef v.ibv_sge *sge = NULL
+ sge = <v.ibv_sge*>malloc(num_sge * sizeof(v.ibv_sge))
+ if sge == NULL:
+ raise PyverbsError('Failed to allocate SGE buffer')
+ for i in range(num_sge):
+ sge[i].addr = sg_list[i].addr
+ sge[i].length = sg_list[i].length
+ sge[i].lkey = sg_list[i].lkey
+ v.ibv_wr_set_sge_list(self.qp_ex, num_sge, sge)
+ free(sge)
+
+ def wr_start(self):
+ v.ibv_wr_start(self.qp_ex)
+
+ def wr_complete(self):
+ rc = v.ibv_wr_complete(self.qp_ex)
+ if rc != 0:
+ raise PyverbsRDMAErrno('ibv_wr_complete failed , returned {}'.
+ format(rc))
+
+ def wr_abort(self):
+ v.ibv_wr_abort(self.qp_ex)
+
+
def _copy_caps(QPCap src, dst):
"""
Copy the QPCaps values of src into the inner ibv_qp_cap struct of dst.
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (5 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 8/8] tests: Add test using the new post send API Noa Osherovich
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
Add a code snippet to demonstrate an extended QP creation using
pyverbs.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
Documentation/pyverbs.md | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md
index c7dddd598303..325e214fca0e 100755
--- a/Documentation/pyverbs.md
+++ b/Documentation/pyverbs.md
@@ -339,6 +339,30 @@ wr = pwr.SendWR()
wr.set_wr_ud(ah, 0x1101, 0) # in real life, use real values
udqp.post_send(wr)
```
+###### Extended QP
+An extended QP exposes a new set of QP send operations to the user -
+extensibility for new send opcodes, vendor specific send opcodes and even vendor
+specific QP types.
+Pyverbs now exposes the needed interface to create such a QP.
+Note that the IBV_QP_INIT_ATTR_SEND_OPS_FLAGS in the `comp_mask` is mandatory
+when using the extended QP's new post send mechanism.
+```python
+from pyverbs.qp import QPCap, QPInitAttrEx, QPAttr, QPEx
+import pyverbs.device as d
+import pyverbs.enums as e
+from pyverbs.pd import PD
+from pyverbs.cq import CQ
+
+
+ctx = d.Context(name='mlx5_0')
+pd = PD(ctx)
+cq = CQ(ctx, 100)
+cap = QPCap(100, 10, 1, 1, 0)
+qia = QPInitAttrEx(qp_type=e.IBV_QPT_UD, scq=cq, rcq=cq, cap=cap, pd=pd,
+ comp_mask=e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS| \
+ e.IBV_QP_INIT_ATTR_PD)
+qp = QPEx(ctx, qia)
+```
##### XRCD
The following code demonstrates creation of an XRCD object.
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH rdma-core 8/8] tests: Add test using the new post send API
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
` (6 preceding siblings ...)
2019-12-31 9:19 ` [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc Noa Osherovich
@ 2019-12-31 9:19 ` Noa Osherovich
7 siblings, 0 replies; 9+ messages in thread
From: Noa Osherovich @ 2019-12-31 9:19 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Noa Osherovich, Edward Srouji
Add simple traffic tests that use the new post_send API.
Currently tested include:
- UD: send, send with immediate
- RC: send, send with immediate, RDMA write, RDMA read, atomic fetch
and add, atomic compare and swap, bind memory window
- XRC: send, send with immediate
The existing traffic methods - traffic() and xrc_traffic() were
modified to support usage of the new API and are now checking whether
a send_op was provided or not.
As RDMA read and write do not require receive WQEs to be posted, an
extra traffic method, rdma_traffic, was added to the tests' utils
section.
Creation of a customized memory region is now available in the tests'
utils section.
Signed-off-by: Noa Osherovich <noaos@mellanox.com>
Reviewed-by: Edward Srouji <edwards@mellanox.com>
---
tests/CMakeLists.txt | 1 +
tests/test_odp.py | 13 +-
tests/test_qpex.py | 295 +++++++++++++++++++++++++++++++++++++++++++
tests/utils.py | 207 +++++++++++++++++++++++++-----
4 files changed, 478 insertions(+), 38 deletions(-)
mode change 100755 => 100644 tests/CMakeLists.txt
create mode 100644 tests/test_qpex.py
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
old mode 100755
new mode 100644
index 6d702425886c..74930f69e19d
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -12,6 +12,7 @@ rdma_python_test(tests
test_mr.py
test_pd.py
test_qp.py
+ test_qpex.py
test_odp.py
test_parent_domain.py
test_rdmacm.py
diff --git a/tests/test_odp.py b/tests/test_odp.py
index d412a7792951..742ad81abb89 100755
--- a/tests/test_odp.py
+++ b/tests/test_odp.py
@@ -1,5 +1,5 @@
+from tests.utils import requires_odp, traffic, xrc_traffic, create_custom_mr
from tests.base import RCResources, UDResources, XRCResources
-from tests.utils import requires_odp, traffic, xrc_traffic
from tests.base import RDMATestCase
from pyverbs.mr import MR
import pyverbs.enums as e
@@ -8,21 +8,20 @@ import pyverbs.enums as e
class OdpUD(UDResources):
@requires_odp('ud')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE,
- e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+ self.mr = create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND,
+ self.msg_size + self.GRH_SIZE)
class OdpRC(RCResources):
@requires_odp('rc')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size,
- e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+ self.mr = create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND)
+
class OdpXRC(XRCResources):
@requires_odp('xrc')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size,
- e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+ self.mr = create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND)
class OdpTestCase(RDMATestCase):
diff --git a/tests/test_qpex.py b/tests/test_qpex.py
new file mode 100644
index 000000000000..922010bce3e5
--- /dev/null
+++ b/tests/test_qpex.py
@@ -0,0 +1,295 @@
+import unittest
+import random
+
+from pyverbs.qp import QPCap, QPInitAttrEx, QPAttr, QPEx, QP
+from pyverbs.pyverbs_error import PyverbsRDMAError
+from pyverbs.mr import MW, MWBindInfo
+from pyverbs.base import inc_rkey
+import pyverbs.enums as e
+from pyverbs.mr import MR
+
+from tests.base import UDResources, RCResources, RDMATestCase, XRCResources
+import tests.utils as u
+
+
+def create_qp_ex(agr_obj, qp_type, send_flags):
+ if qp_type == e.IBV_QPT_XRC_SEND:
+ cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0,
+ max_send_sge=1)
+ else:
+ cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs,
+ max_recv_sge=1, max_send_sge=1)
+ qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq,
+ rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags,
+ comp_mask=e.IBV_QP_INIT_ATTR_PD |
+ e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
+ qp_attr = QPAttr(port_num=agr_obj.ib_port)
+ if qp_type == e.IBV_QPT_UD:
+ qp_attr.qkey = agr_obj.UD_QKEY
+ qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
+ if qp_type == e.IBV_QPT_RC:
+ qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+ e.IBV_ACCESS_REMOTE_READ | \
+ e.IBV_ACCESS_REMOTE_ATOMIC
+ try:
+ # We don't have capability bits for this
+ qp = QPEx(agr_obj.ctx, qia, qp_attr)
+ except PyverbsRDMAError as exp:
+ if 'Operation not supported' in exp.args[0]:
+ raise unittest.SkipTest('Extended QP not supported on this device')
+ raise exp
+ return qp
+
+
+class QpExUDSend(UDResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND)
+
+
+class QpExRCSend(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND)
+
+
+class QpExXRCSend(XRCResources):
+ def create_qp(self):
+ qp_attr = QPAttr(port_num=self.ib_port)
+ qp_attr.pkey_index = 0
+ for _ in range(self.qp_count):
+ attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
+ comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
+ xrcd=self.xrcd)
+ qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+ e.IBV_ACCESS_REMOTE_READ
+ recv_qp = QP(self.ctx, attr_ex, qp_attr)
+ self.rqp_lst.append(recv_qp)
+
+ send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND, e.IBV_QP_EX_WITH_SEND)
+ self.sqp_lst.append(send_qp)
+ self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
+ self.psns.append(random.getrandbits(24))
+
+
+class QpExUDSendImm(UDResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+
+class QpExRCSendImm(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+
+class QpExXRCSendImm(XRCResources):
+ def create_qp(self):
+ qp_attr = QPAttr(port_num=self.ib_port)
+ qp_attr.pkey_index = 0
+ for _ in range(self.qp_count):
+ attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
+ comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
+ xrcd=self.xrcd)
+ qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+ e.IBV_ACCESS_REMOTE_READ
+ recv_qp = QP(self.ctx, attr_ex, qp_attr)
+ self.rqp_lst.append(recv_qp)
+
+ send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND,
+ e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+ self.sqp_lst.append(send_qp)
+ self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
+ self.psns.append(random.getrandbits(24))
+
+
+class QpExRCRDMAWrite(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_WRITE)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
+
+
+class QpExRCRDMAWriteImm(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC,
+ e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
+
+
+class QpExRCRDMARead(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_READ)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_READ)
+
+
+class QpExRCAtomicCmpSwp(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC,
+ e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_ATOMIC)
+
+
+class QpExRCAtomicFetchAdd(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC,
+ e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_ATOMIC)
+
+
+class QpExRCBindMw(RCResources):
+ def create_qp(self):
+ self.qp = create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_BIND_MW)
+
+ def create_mr(self):
+ self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
+
+
+class QpExTestCase(RDMATestCase):
+ """ Run traffic using the new post send API. """
+ def setUp(self):
+ super().setUp()
+ self.iters = 100
+ self.qp_dict = {'ud_send': QpExUDSend, 'rc_send': QpExRCSend,
+ 'xrc_send': QpExXRCSend, 'ud_send_imm': QpExUDSendImm,
+ 'rc_send_imm': QpExRCSendImm,
+ 'xrc_send_imm': QpExXRCSendImm,
+ 'rc_write': QpExRCRDMAWrite,
+ 'rc_write_imm': QpExRCRDMAWriteImm,
+ 'rc_read': QpExRCRDMARead,
+ 'rc_cmp_swp': QpExRCAtomicCmpSwp,
+ 'rc_fetch_add': QpExRCAtomicFetchAdd,
+ 'rc_bind_mw': QpExRCBindMw}
+
+ def create_players(self, qp_type):
+ client = self.qp_dict[qp_type](self.dev_name, self.ib_port,
+ self.gid_index)
+ server = self.qp_dict[qp_type](self.dev_name, self.ib_port,
+ self.gid_index)
+ if 'xrc' in qp_type:
+ client.pre_run(server.psns, server.qps_num)
+ server.pre_run(client.psns, client.qps_num)
+ else:
+ client.pre_run(server.psn, server.qpn)
+ server.pre_run(client.psn, client.qpn)
+ return client, server
+
+ def test_qp_ex_ud_send(self):
+ client, server = self.create_players('ud_send')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND)
+
+ def test_qp_ex_rc_send(self):
+ client, server = self.create_players('rc_send')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND)
+
+ def test_qp_ex_xrc_send(self):
+ client, server = self.create_players('xrc_send')
+ u.xrc_traffic(client, server, send_op=e.IBV_QP_EX_WITH_SEND)
+
+ def test_qp_ex_ud_send_imm(self):
+ client, server = self.create_players('ud_send_imm')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+ def test_qp_ex_rc_send_imm(self):
+ client, server = self.create_players('rc_send_imm')
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+ def test_qp_ex_xrc_send_imm(self):
+ client, server = self.create_players('xrc_send_imm')
+ u.xrc_traffic(client, server, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+
+ def test_qp_ex_rc_rdma_write(self):
+ client, server = self.create_players('rc_write')
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE)
+
+ def test_qp_ex_rc_rdma_write_imm(self):
+ client, server = self.create_players('rc_write_imm')
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
+
+ def test_qp_ex_rc_rdma_read(self):
+ client, server = self.create_players('rc_read')
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ server.mr.write('s' * server.msg_size, server.msg_size)
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_READ)
+
+ def test_qp_ex_rc_atomic_cmp_swp(self):
+ client, server = self.create_players('rc_cmp_swp')
+ client.msg_size = 8 # Atomic work on 64b operators
+ server.msg_size = 8
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ server.mr.write('s' * 8, 8)
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
+
+ def test_qp_ex_rc_atomic_fetch_add(self):
+ client, server = self.create_players('rc_fetch_add')
+ client.msg_size = 8 # Atomic work on 64b operators
+ server.msg_size = 8
+ client.rkey = server.mr.rkey
+ server.rkey = client.mr.rkey
+ client.raddr = server.mr.buf
+ server.raddr = client.mr.buf
+ server.mr.write('s' * 8, 8)
+ u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+ is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
+
+ def test_qp_ex_rc_bind_mw(self):
+ """
+ Verify bind memory window operation using the new post_send API.
+ Instead of checking through regular pingpong style traffic, we'll
+ do as follows:
+ - Register an MR with remote write access
+ - Bind a MW without remote write permission to the MR
+ - Verify that remote write fails
+ Since it's a unique flow, it's an integral part of that test rather
+ than a utility method.
+ """
+ client, server = self.create_players('rc_bind_mw')
+ client_sge = u.get_send_element(client, False)[1]
+ # Create a MW and bind it
+ server.qp.wr_start()
+ server.qp.wr_id = 0x123
+ server.qp.wr_flags = e.IBV_SEND_SIGNALED
+ bind_info = MWBindInfo(server.mr, server.mr.buf, server.mr.length,
+ e.IBV_ACCESS_LOCAL_WRITE)
+ mw = MW(server.pd, mw_type=e.IBV_MW_TYPE_2)
+ new_key = inc_rkey(server.mr.rkey)
+ server.qp.wr_bind_mw(mw, new_key, bind_info)
+ server.qp.wr_complete()
+ u.poll_cq(server.cq)
+ # Verify remote write fails
+ client.qp.wr_start()
+ client.qp.wr_id = 0x124
+ client.qp.wr_flags = e.IBV_SEND_SIGNALED
+ client.qp.wr_rdma_write(new_key, server.mr.buf)
+ client.qp.wr_set_sge(client_sge)
+ client.qp.wr_complete()
+ try:
+ u.poll_cq(client.cq)
+ except PyverbsRDMAError as exp:
+ if 'Completion status is Remote access error' not in exp.args[0]:
+ raise exp
+
diff --git a/tests/utils.py b/tests/utils.py
index 47eacfee35e5..20132a7cf40b 100755
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -7,6 +7,7 @@ from itertools import combinations as com
from string import ascii_lowercase as al
import unittest
import random
+import socket
from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError
from pyverbs.addr import AHAttr, AH, GlobalRoute
@@ -16,6 +17,7 @@ from tests.base import XRCResources
from pyverbs.cq import PollCqAttr
import pyverbs.device as d
import pyverbs.enums as e
+from pyverbs.mr import MR
MAX_MR_SIZE = 4194304
# Some HWs limit DM address and length alignment to 4 for read and write
@@ -29,6 +31,7 @@ MAX_DM_LOG_ALIGN = 6
# Raw Packet QP supports TSO header, which creates a larger send WQE.
MAX_RAW_PACKET_SEND_WR = 2500
GRH_SIZE = 40
+IMM_DATA = 1234
def get_mr_length():
@@ -197,7 +200,7 @@ def random_qp_create_flags(qpt, attr_ex):
def random_qp_init_attr_ex(attr_ex, attr, qpt=None):
"""
- Create a random-valued QPInitAttrEX object with the given QP type.
+ Create a random-valued QPInitAttrEx object with the given QP type.
QP type affects QP capabilities, so allow users to set it and still get
valid attributes.
:param attr_ex: Extended device attributes for capability checks
@@ -251,24 +254,38 @@ def wc_status_to_str(status):
except KeyError:
return 'Unknown WC status ({s})'.format(s=status)
+
+def create_custom_mr(agr_obj, additional_access_flags=0, size=None):
+ """
+ Creates a memory region using the aggregation object's PD.
+ If size is None, the agr_obj's message size is used to set the MR's size.
+ The access flags are local write and the additional_access_flags.
+ :param agr_obj: The aggregation object that creates the MR
+ :param additional_access_flags: Addition access flags to set in the MR
+ :param size: MR's length. If None, agr_obj.msg_size is used.
+ """
+ mr_length = size if size else agr_obj.msg_size
+ return MR(agr_obj.pd, mr_length,
+ e.IBV_ACCESS_LOCAL_WRITE | additional_access_flags)
+
# Traffic helpers
-def get_send_wr(agr_obj, is_server):
+def get_send_element(agr_obj, is_server):
"""
- Creates a single SGE Send WR for agr_obj's QP type. The content of the
- message is either 's' for server side or 'c' for client side.
+ Creates a single SGE and a single Send WR for agr_obj's QP type. The content
+ of the message is either 's' for server side or 'c' for client side.
:param agr_obj: Aggregation object which contains all resources necessary
:param is_server: Indicates whether this is server or client side
- :return: send wr
+ :return: send wr and its SGE
"""
+ mr = agr_obj.mr
qp_type = agr_obj.sqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
else agr_obj.qp.qp_type
- mr = agr_obj.mr
offset = GRH_SIZE if qp_type == e.IBV_QPT_UD else 0
- send_sge = SGE(mr.buf + offset, agr_obj.msg_size, mr.lkey)
msg = (agr_obj.msg_size + offset) * ('s' if is_server else 'c')
mr.write(msg, agr_obj.msg_size + offset)
- return SendWR(num_sge=1, sg=[send_sge])
+ sge = SGE(mr.buf + offset, agr_obj.msg_size, mr.lkey)
+ return SendWR(num_sge=1, sg=[sge]), sge
def get_recv_wr(agr_obj):
@@ -286,6 +303,64 @@ def get_recv_wr(agr_obj):
return RecvWR(sg=[recv_sge], num_sge=1)
+def get_global_ah(agr_obj, gid_index, port):
+ gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
+ sgid_index=gid_index)
+ ah_attr = AHAttr(port_num=port, is_global=1, gr=gr,
+ dlid=agr_obj.port_attr.lid)
+ return AH(agr_obj.pd, attr=ah_attr)
+
+
+def xrc_post_send(agr_obj, qp_num, send_object, gid_index, port, send_op=None):
+ agr_obj.qp = agr_obj.sqp_lst[qp_num]
+ if send_op:
+ post_send_ex(agr_obj, send_object, gid_index, port, send_op)
+ else:
+ post_send(agr_obj, send_object, gid_index, port)
+
+
+def post_send_ex(agr_obj, send_object, gid_index, port, send_op=None):
+ qp_type = agr_obj.qp.qp_type
+ agr_obj.qp.wr_start()
+ agr_obj.qp.wr_id = 0x123
+ agr_obj.qp.wr_flags = e.IBV_SEND_SIGNALED
+ if send_op == e.IBV_QP_EX_WITH_SEND:
+ agr_obj.qp.wr_send()
+ elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE:
+ agr_obj.qp.wr_rdma_write(agr_obj.rkey, agr_obj.raddr)
+ elif send_op == e.IBV_QP_EX_WITH_SEND_WITH_IMM:
+ agr_obj.qp.wr_send_imm(IMM_DATA)
+ elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM:
+ agr_obj.qp.wr_rdma_write_imm(agr_obj.rkey, agr_obj.raddr, IMM_DATA)
+ elif send_op == e.IBV_QP_EX_WITH_RDMA_READ:
+ agr_obj.qp.wr_rdma_read(agr_obj.rkey, agr_obj.raddr)
+ elif send_op == e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP:
+ # We're checking the returned value (remote's content), so cmp/swp
+ # values are of no importance.
+ agr_obj.qp.wr_atomic_cmp_swp(agr_obj.rkey, agr_obj.raddr, 42, 43)
+ elif send_op == e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD:
+ agr_obj.qp.wr_atomic_fetch_add(agr_obj.rkey, agr_obj.raddr, 1)
+ elif send_op == e.IBV_QP_EX_WITH_BIND_MW:
+ bind_info = MWBindInfo(agr_obj.mr, agr_obj.mr.buf, agr_obj.mr.rkey,
+ e.IBV_ACCESS_REMOTE_WRITE)
+ mw = MW(agr_obj.pd, mw_type=e.IBV_MW_TYPE_2)
+ # A new rkey is needed to be set into bind_info, modify rkey
+ agr_obj.qp.wr_bind_mw(mw, agr_obj.mr.rkey + 12, bind_info)
+ agr_obj.qp.wr_complete()
+ return
+ #agr_obj.qp.wr_start()
+ #agr_obj.qp.wr_id = 0x123
+ #agr_obj.qp.wr_flags = e.IBV_SEND_SIGNALED
+ #agr_obj.qp.wr_send()
+ if qp_type == e.IBV_QPT_UD:
+ ah = get_global_ah(agr_obj, gid_index, port)
+ agr_obj.qp.wr_set_ud_addr(ah, agr_obj.rqpn, agr_obj.UD_QKEY)
+ if qp_type == e.IBV_QPT_XRC_SEND:
+ agr_obj.qp.wr_set_xrc_srqn(agr_obj.remote_srqn)
+ agr_obj.qp.wr_set_sge(send_object)
+ agr_obj.qp.wr_complete()
+
+
def post_send(agr_obj, send_wr, gid_index, port):
"""
Post a single send WR to the QP. Post_send's second parameter (send bad wr)
@@ -299,11 +374,7 @@ def post_send(agr_obj, send_wr, gid_index, port):
"""
qp_type = agr_obj.qp.qp_type
if qp_type == e.IBV_QPT_UD:
- gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
- sgid_index=gid_index)
- ah_attr = AHAttr(port_num=port, is_global=1, gr=gr,
- dlid=agr_obj.port_attr.lid)
- ah = AH(agr_obj.pd, attr=ah_attr)
+ ah = get_global_ah(agr_obj, gid_index, port)
send_wr.set_wr_ud(ah, agr_obj.rqpn, agr_obj.UD_QKEY)
agr_obj.qp.post_send(send_wr, None)
@@ -321,7 +392,7 @@ def post_recv(qp, recv_wr, num_wqes=1):
qp.post_recv(recv_wr, None)
-def poll_cq(cq, count=1):
+def poll_cq(cq, count=1, data=None):
"""
Poll <count> completions from the CQ.
Note: This function calls the blocking poll() method of the CQ
@@ -329,6 +400,8 @@ def poll_cq(cq, count=1):
single CQ event when events are used.
:param cq: CQ to poll from
:param count: How many completions to poll
+ :param data: In case of a work request with immediate, the immediate data
+ to be compared after poll
:return: An array of work completions of length <count>, None
when events are used
"""
@@ -339,15 +412,21 @@ def poll_cq(cq, count=1):
if wc.status != e.IBV_WC_SUCCESS:
raise PyverbsRDMAError('Completion status is {s}'.
format(s=wc_status_to_str(wc.status)))
+ if data:
+ if wc.wc_flags & e.IBV_WC_WITH_IMM == 0:
+ raise PyverbsRDMAError('Completion without immediate')
+ assert socket.ntohl(wc.imm_data) == data
count -= nc
return wcs
-def poll_cq_ex(cqex, count=1):
+def poll_cq_ex(cqex, count=1, data=None):
"""
Poll <count> completions from the extended CQ.
:param cq: CQEX to poll from
:param count: How many completions to poll
+ :param data: In case of a work request with immediate, the immediate data
+ to be compared after poll
:return: None
"""
poll_attr = PollCqAttr()
@@ -360,6 +439,8 @@ def poll_cq_ex(cqex, count=1):
if cqex.status != e.IBV_WC_SUCCESS:
raise PyverbsRDMAErrno('Completion status is {s}'.
format(s=cqex.status))
+ if data:
+ assert data == socket.ntohl(cqex.read_imm_data())
# Now poll the rest of the packets
while count > 0:
ret = cqex.poll_next()
@@ -370,6 +451,8 @@ def poll_cq_ex(cqex, count=1):
if cqex.status != e.IBV_WC_SUCCESS:
raise PyverbsRDMAErrno('Completion status is {s}'.
format(s=cqex.status))
+ if data:
+ assert data == socket.ntohl(cqex.read_imm_data())
count -= 1
cqex.end_poll()
@@ -398,7 +481,13 @@ def validate(received_str, is_server, msg_size):
format(exp=expected_str, rcv=received_str))
-def traffic(client, server, iters, gid_idx, port, is_cq_ex=False):
+def send(agr_obj, send_wr, gid_index, port, send_op=None):
+ if send_op:
+ return post_send_ex(agr_obj, send_wr, gid_index, port, send_op)
+ return post_send(agr_obj, send_wr, gid_index, port)
+
+
+def traffic(client, server, iters, gid_idx, port, is_cq_ex=False, send_op=None):
"""
Runs basic traffic between two sides
:param client: client side, clients base class is BaseTraffic
@@ -407,32 +496,83 @@ def traffic(client, server, iters, gid_idx, port, is_cq_ex=False):
:param gid_idx: local gid index
:param port: IB port
:param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
+ :param send_op: If not None, new post send API is assumed.
:return:
"""
poll = poll_cq_ex if is_cq_ex else poll_cq
+ if send_op == e.IBV_QP_EX_WITH_SEND_WITH_IMM or \
+ send_op == e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM:
+ imm_data = IMM_DATA
+ else:
+ imm_data = None
+ # Using the new post send API, we need the SGE, not the SendWR
+ send_element_idx = 1 if send_op else 0
s_recv_wr = get_recv_wr(server)
c_recv_wr = get_recv_wr(client)
post_recv(client.qp, c_recv_wr, client.num_msgs)
post_recv(server.qp, s_recv_wr, server.num_msgs)
read_offset = GRH_SIZE if client.qp.qp_type == e.IBV_QPT_UD else 0
for _ in range(iters):
- c_send_wr = get_send_wr(client, False)
- post_send(client, c_send_wr, gid_idx, port)
+ c_send_wr = get_send_element(client, False)[send_element_idx]
+ send(client, c_send_wr, gid_idx, port, send_op)
poll(client.cq)
- poll(server.cq)
+ poll(server.cq, data=imm_data)
post_recv(server.qp, s_recv_wr)
msg_received = server.mr.read(server.msg_size, read_offset)
validate(msg_received, True, server.msg_size)
- s_send_wr = get_send_wr(server, True)
- post_send(server, s_send_wr, gid_idx, port)
+ s_send_wr = get_send_element(server, True)[send_element_idx]
+ send(server, s_send_wr, gid_idx, port, send_op)
poll(server.cq)
- poll(client.cq)
+ poll(client.cq, data=imm_data)
post_recv(client.qp, c_recv_wr)
msg_received = client.mr.read(client.msg_size, read_offset)
validate(msg_received, False, client.msg_size)
-def xrc_traffic(client, server, is_cq_ex=False):
+def rdma_traffic(client, server, iters, gid_idx, port, is_cq_ex=False, send_op=None):
+ """
+ Runs basic RDMA traffic between two sides. No receive WQEs are posted. For
+ RDMA send with immediate, use traffic().
+ :param client: client side, clients base class is BaseTraffic
+ :param server: server side, servers base class is BaseTraffic
+ :param iters: number of traffic iterations
+ :param gid_idx: local gid index
+ :param port: IB port
+ :param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
+ :param send_op: If not None, new post send API is assumed.
+ :return:
+ """
+ # Using the new post send API, we need the SGE, not the SendWR
+ send_element_idx = 1 if send_op else 0
+ same_side_check = (send_op == e.IBV_QP_EX_WITH_RDMA_READ or
+ send_op == e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP or
+ send_op == e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
+ for _ in range(iters):
+ c_send_wr = get_send_element(client, False)[send_element_idx]
+ send(client, c_send_wr, gid_idx, port, send_op)
+ poll_cq(client.cq)
+ if same_side_check:
+ msg_received = client.mr.read(client.msg_size, 0)
+ else:
+ msg_received = server.mr.read(server.msg_size, 0)
+ validate(msg_received, False if same_side_check else True,
+ server.msg_size)
+ s_send_wr = get_send_element(server, True)[send_element_idx]
+ if same_side_check:
+ client.mr.write('c' * client.msg_size, client.msg_size)
+ send(server, s_send_wr, gid_idx, port, send_op)
+ poll_cq(server.cq)
+ if same_side_check:
+ msg_received = server.mr.read(client.msg_size, 0)
+ else:
+ msg_received = client.mr.read(server.msg_size, 0)
+ validate(msg_received, True if same_side_check else False,
+ client.msg_size)
+ if same_side_check:
+ server.mr.write('s' * server.msg_size, server.msg_size)
+
+
+def xrc_traffic(client, server, is_cq_ex=False, send_op=None):
"""
Runs basic xrc traffic, this function assumes that number of QPs, which
server and client have are equal, server.send_qp[i] is connected to
@@ -444,27 +584,32 @@ def xrc_traffic(client, server, is_cq_ex=False):
:param server: Aggregation object of the passive side, should be an instance
of XRCResources class
:param is_cq_ex: If True, use poll_cq_ex() rather than poll_cq()
+ :param send_op: If not None, new post send API is assumed.
:return: None
"""
poll = poll_cq_ex if is_cq_ex else poll_cq
- client_srqn = client.srq.get_srq_num()
- server_srqn = server.srq.get_srq_num()
+ server.remote_srqn = client.srq.get_srq_num()
+ client.remote_srqn = server.srq.get_srq_num()
s_recv_wr = get_recv_wr(server)
c_recv_wr = get_recv_wr(client)
post_recv(client.srq, c_recv_wr, client.qp_count*client.num_msgs)
post_recv(server.srq, s_recv_wr, server.qp_count*server.num_msgs)
+ # Using the new post send API, we need the SGE, not the SendWR
+ send_element_idx = 1 if send_op else 0
for _ in range(client.num_msgs):
for i in range(server.qp_count):
- c_send_wr = get_send_wr(client, False)
- c_send_wr.set_qp_type_xrc(server_srqn)
- client.sqp_lst[i].post_send(c_send_wr)
+ c_send_wr = get_send_element(client, False)[send_element_idx]
+ if send_op is None:
+ c_send_wr.set_qp_type_xrc(client.remote_srqn)
+ xrc_post_send(client, i, c_send_wr, 0, 0, send_op)
poll(client.cq)
poll(server.cq)
msg_received = server.mr.read(server.msg_size, 0)
validate(msg_received, True, server.msg_size)
- s_send_wr = get_send_wr(server, True)
- s_send_wr.set_qp_type_xrc(client_srqn)
- server.sqp_lst[i].post_send(s_send_wr)
+ s_send_wr = get_send_element(server, True)[send_element_idx]
+ if send_op is None:
+ s_send_wr.set_qp_type_xrc(server.remote_srqn)
+ xrc_post_send(server, i, s_send_wr, 0, 0, send_op)
poll(server.cq)
poll(client.cq)
msg_received = client.mr.read(client.msg_size, 0)
--
2.21.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
end of thread, other threads:[~2019-12-31 9:20 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-12-31 9:19 [PATCH rdma-core 0/8] pyverbs: Add support for the new post send API Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 1/8] pyverbs: Add support for memory window Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 2/8] pyverbs: Add TSO support Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 3/8] tests: Decrease maximal TSO header size Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 4/8] tests: Use post_recv in the right place Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 5/8] pyverbs: Expose MR's length property Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 6/8] pyverbs: Introduce extended QP and new post send Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 7/8] Documentation: Add extended QP to pyverbs's doc Noa Osherovich
2019-12-31 9:19 ` [PATCH rdma-core 8/8] tests: Add test using the new post send API Noa Osherovich
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).