* [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs
@ 2019-11-04 10:37 Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 1/5] pyverbs: New CMID class Noa Osherovich
` (4 more replies)
0 siblings, 5 replies; 8+ messages in thread
From: Noa Osherovich @ 2019-11-04 10:37 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Noa Osherovich
The following patches add support for RDMACM in pyverbs. Currently
only synchronous data path is supported (creation using rdma_create_ep).
Testing infrastructure for RDMACM is also added as well as a synchronous
traffic test.
Maxim Chicherin (5):
pyverbs: New CMID class
tests: Fix PD API test
tests: New CMResources Class
tests: Add RDMACM synchronous traffic test
Documentation: Document creation of CMID
Documentation/pyverbs.md | 30 ++++
buildlib/pyverbs_functions.cmake | 2 +-
pyverbs/CMakeLists.txt | 2 +
pyverbs/cm_enums.pyx | 1 +
pyverbs/cmid.pxd | 25 +++
pyverbs/cmid.pyx | 285 +++++++++++++++++++++++++++++++
pyverbs/device.pyx | 15 +-
pyverbs/librdmacm.pxd | 106 ++++++++++++
pyverbs/librdmacm_enums.pxd | 32 ++++
pyverbs/pd.pyx | 24 ++-
tests/CMakeLists.txt | 2 +
tests/base.py | 52 ++++++
tests/rdmacm_utils.py | 43 +++++
tests/test_pd.py | 3 +-
tests/test_rdmacm.py | 57 +++++++
15 files changed, 668 insertions(+), 11 deletions(-)
create mode 120000 pyverbs/cm_enums.pyx
create mode 100755 pyverbs/cmid.pxd
create mode 100755 pyverbs/cmid.pyx
create mode 100755 pyverbs/librdmacm.pxd
create mode 100755 pyverbs/librdmacm_enums.pxd
mode change 100644 => 100755 pyverbs/pd.pyx
mode change 100644 => 100755 tests/CMakeLists.txt
create mode 100755 tests/rdmacm_utils.py
mode change 100644 => 100755 tests/test_pd.py
create mode 100755 tests/test_rdmacm.py
--
2.21.0
^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH rdma-core 1/5] pyverbs: New CMID class
2019-11-04 10:37 [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs Noa Osherovich
@ 2019-11-04 10:37 ` Noa Osherovich
2019-11-04 11:09 ` Benjamin Drung
2019-11-04 10:37 ` [PATCH rdma-core 2/5] tests: Fix PD API test Noa Osherovich
` (3 subsequent siblings)
4 siblings, 1 reply; 8+ messages in thread
From: Noa Osherovich @ 2019-11-04 10:37 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Maxim Chicherin
From: Maxim Chicherin <maximc@mellanox.com>
The CMID class represents rdma_cm_id, RDMA Connection Manager.
Currently only synchronous data path is supported. Support was added
to Context and PD classes to allow creation using rdmacm's API.
Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
buildlib/pyverbs_functions.cmake | 2 +-
pyverbs/CMakeLists.txt | 2 +
pyverbs/cm_enums.pyx | 1 +
pyverbs/cmid.pxd | 25 +++
pyverbs/cmid.pyx | 285 +++++++++++++++++++++++++++++++
pyverbs/device.pyx | 15 +-
pyverbs/librdmacm.pxd | 106 ++++++++++++
pyverbs/librdmacm_enums.pxd | 32 ++++
pyverbs/pd.pyx | 24 ++-
9 files changed, 483 insertions(+), 9 deletions(-)
create mode 120000 pyverbs/cm_enums.pyx
create mode 100755 pyverbs/cmid.pxd
create mode 100755 pyverbs/cmid.pyx
create mode 100755 pyverbs/librdmacm.pxd
create mode 100755 pyverbs/librdmacm_enums.pxd
mode change 100644 => 100755 pyverbs/pd.pyx
diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake
index 4c255054fe94..a494ec16610b 100644
--- a/buildlib/pyverbs_functions.cmake
+++ b/buildlib/pyverbs_functions.cmake
@@ -25,7 +25,7 @@ function(rdma_cython_module PY_MODULE LINKER_FLAGS)
COMPILE_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option ${NO_VAR_TRACKING_FLAGS}"
LIBRARY_OUTPUT_DIRECTORY "${BUILD_PYTHON}/${PY_MODULE}"
PREFIX "")
- target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs ${LINKER_FLAGS})
+ target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs rdmacm ${LINKER_FLAGS})
install(TARGETS ${SONAME}
DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
endforeach()
diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt
index 7bbb5fc841c0..de37025ce324 100755
--- a/pyverbs/CMakeLists.txt
+++ b/pyverbs/CMakeLists.txt
@@ -4,6 +4,8 @@
rdma_cython_module(pyverbs ""
addr.pyx
base.pyx
+ cm_enums.pyx
+ cmid.pyx
cq.pyx
device.pyx
enums.pyx
diff --git a/pyverbs/cm_enums.pyx b/pyverbs/cm_enums.pyx
new file mode 120000
index 000000000000..bdab2b585a1d
--- /dev/null
+++ b/pyverbs/cm_enums.pyx
@@ -0,0 +1 @@
+librdmacm_enums.pxd
\ No newline at end of file
diff --git a/pyverbs/cmid.pxd b/pyverbs/cmid.pxd
new file mode 100755
index 000000000000..56bc755daf42
--- /dev/null
+++ b/pyverbs/cmid.pxd
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from pyverbs.base cimport PyverbsObject, PyverbsCM
+from libc.string cimport memcpy, memset
+from libc.stdlib cimport free, malloc
+cimport pyverbs.librdmacm as cm
+
+
+cdef class CMID(PyverbsCM):
+ cdef cm.rdma_cm_id *id
+ cdef object ctx
+ cdef object pd
+ cpdef close(self)
+
+
+cdef class AddrInfo(PyverbsObject):
+ cdef cm.rdma_addrinfo *addr_info
+ cpdef close(self)
+
+
+cdef class ConnParam(PyverbsObject):
+ cdef cm.rdma_conn_param conn_param
\ No newline at end of file
diff --git a/pyverbs/cmid.pyx b/pyverbs/cmid.pyx
new file mode 100755
index 000000000000..c752feda8781
--- /dev/null
+++ b/pyverbs/cmid.pyx
@@ -0,0 +1,285 @@
+from pyverbs.pyverbs_error import PyverbsError
+from pyverbs.device cimport PortAttr, Context
+from pyverbs.qp cimport QPInitAttr, QPAttr
+from pyverbs.base import PyverbsRDMAErrno
+cimport pyverbs.libibverbs_enums as e
+cimport pyverbs.librdmacm_enums as ce
+cimport pyverbs.libibverbs as v
+cimport pyverbs.librdmacm as cm
+from pyverbs.pd cimport PD
+from pyverbs.mr cimport MR
+from pyverbs.cq cimport WC
+
+
+cdef class ConnParam(PyverbsObject):
+
+ def __cinit__(self, resources=1, depth=1, flow_control=0, retry=5,
+ rnr_retry=5, srq=0, qp_num=0):
+ """
+ Initialize a ConnParam object over an underlying rdma_conn_param
+ C object which contains connection parameters. There are a few types of
+ port spaces in RDMACM: RDMA_PS_TCP, RDMA_PS_UDP, RDMA_PS_IB and
+ RDMA_PS_IPOIB. RDMA_PS_TCP resembles RC QP connection, which provides
+ reliable, connection-oriented QP communication. This object applies only
+ to RDMA_PS_TCP port space.
+ :param resources: Max outstanding RDMA read and atomic ops that local
+ side will accept from the remote side.
+ :param depth: Max outstanding RDMA read and atomic ops that local side
+ will have to the remote side.
+ :param flow_control: Specifies if hardware flow control is available.
+ :param retry: Max number of times that a send, RDMA or atomic op from
+ the remote peer should be retried.
+ :param rnr_retry: The maximum number of times that a send operation from
+ the remote peer should be retried on a connection
+ after receiving a receiver not ready (RNR) error.
+ :param srq: Specifies if the QP using shared receive queue, ignored if
+ the QP created by CMID.
+ :param qp_num: Specifies the QP number, ignored if the QP created by
+ CMID.
+ :return: ConnParam object
+ """
+ memset(&self.conn_param, 0, sizeof(cm.rdma_conn_param))
+ self.conn_param.responder_resources = resources
+ self.conn_param.initiator_depth = depth
+ self.conn_param.flow_control = flow_control
+ self.conn_param.retry_count = retry
+ self.conn_param.rnr_retry_count = rnr_retry
+ self.conn_param.srq = srq
+ self.conn_param.qp_num = qp_num
+
+ def __str__(self):
+ print_format = '{:<4}: {:<4}\n'
+ return '{}: {}\n'.format('Connection parameters', "") +\
+ print_format.format('responder resources', self.conn_param.responder_resources) +\
+ print_format.format('initiator depth', self.conn_param.initiator_depth) +\
+ print_format.format('flow control', self.conn_param.flow_control) +\
+ print_format.format('retry count', self.conn_param.retry_count) +\
+ print_format.format('rnr retry count', self.conn_param.rnr_retry_count) +\
+ print_format.format('srq', self.conn_param.srq) +\
+ print_format.format('qp number', self.conn_param.qp_num)
+
+
+cdef class AddrInfo(PyverbsObject):
+ def __cinit__(self, node=None, service=None, port_space=0, flags=0):
+ """
+ Initialize an AddrInfo object over an underlying rdma_addrinfo C object.
+ :param node: Name, dotted-decimal IPv4 or IPv6 hex address to resolve.
+ :param service: The service name or port number of the address.
+ :param port_space: RDMA port space used (RDMA_PS_UDP or RDMA_PS_TCP).
+ :param flags: Hint flags which control the operation.
+ :return: An AddrInfo object which contains information needed to
+ establish communication.
+ """
+ cdef char* srvc = NULL
+ cdef char* address = NULL
+ cdef cm.rdma_addrinfo hints
+ cdef cm.rdma_addrinfo *hints_ptr = NULL
+
+ if node is not None:
+ node = node.encode('utf-8')
+ address = <char*>node
+ if service is not None:
+ service = service.encode('utf-8')
+ srvc = <char*>service
+ if port_space != 0:
+ hints_ptr = &hints
+ memset(hints_ptr, 0, sizeof(cm.rdma_addrinfo))
+ hints.ai_port_space = port_space
+ hints.ai_flags = flags
+ ret = cm.rdma_getaddrinfo(address, srvc, hints_ptr, &self.addr_info)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to get Address Info')
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ self.logger.debug('Closing AddrInfo')
+ if self.addr_info != NULL:
+ cm.rdma_freeaddrinfo(self.addr_info)
+ self.addr_info = NULL
+
+
+cdef class CMID(PyverbsCM):
+
+ def __cinit__(self, object creator=None, QPInitAttr qp_init_attr=None,
+ PD pd=None):
+ """
+ Initialize a CMID object over an underlying rdma_cm_id C object.
+ This is the main RDMA CM object which provides most of the rdmacm API.
+ Currently only synchronous RDMA_PS_TCP communication supported.
+ :param creator: For synchronous communication we need AddrInfo object in
+ order to establish connection. We allow creator to be
+ None for inner usage, see get_request method.
+ :param pd: Optional parameter, a PD to be associated with this CMID.
+ :param qp_init_attr: Optional initial QP attributes of CMID
+ associated QP.
+ :return: CMID object for synchronous communication.
+ """
+ cdef v.ibv_qp_init_attr *init
+ cdef v.ibv_pd *in_pd = NULL
+ self.pd = None
+ self.ctx = None
+ if creator is None:
+ return
+ elif issubclass(type(creator), AddrInfo):
+ init = NULL if qp_init_attr is None else &qp_init_attr.attr
+ if pd is not None:
+ in_pd = pd.pd
+ self.pd = pd
+ ret = cm.rdma_create_ep(&self.id, (<AddrInfo>creator).addr_info,
+ in_pd, init)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to create CM ID')
+ if not (<AddrInfo>creator).addr_info.ai_flags & ce.RAI_PASSIVE:
+ self.ctx = Context(cmid=self)
+ if self.pd is None:
+ self.pd = PD(self)
+ else:
+ raise PyverbsRDMAErrno('Cannot create CM ID from {obj}'
+ .format(obj=type(creator)))
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ self.logger.debug('Closing CMID')
+ if self.id != NULL:
+ cm.rdma_destroy_ep(self.id)
+ if self.ctx:
+ (<Context>self.ctx).context = NULL
+ if self.pd:
+ (<PD>self.pd).pd = NULL
+ self.id = NULL
+
+ def get_request(self):
+ """
+ Retrieves the next pending connection request event. The call may only
+ be used on listening CMIDs operating synchronously. If the call is
+ successful, a new CMID representing the connection request will be
+ returned to the user. The new CMID will reference event information
+ associated with the request until the user calls reject, accept, or
+ close on the newly created identifier.
+ :return: New CMID representing the connection request.
+ """
+ to_conn = CMID()
+ ret = cm.rdma_get_request(self.id, &to_conn.id)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to get request, no connection established')
+ self.ctx = Context(cmid=to_conn)
+ self.pd = PD(to_conn)
+ return to_conn
+
+ def reg_msgs(self, size):
+ """
+ Registers a memory region for sending or receiving messages or for
+ RDMA operations. The registered memory may then be posted to an CMID
+ using post_send or post_recv methods.
+ :param size: The total length of the memory to register
+ :return: registered MR
+ """
+ return MR(self.pd, size, e.IBV_ACCESS_LOCAL_WRITE)
+
+ def listen(self, backlog=0):
+ """
+ Listen for incoming connection requests or datagram service lookup.
+ The listen is restricted to the locally bound source address.
+ :param backlog: The backlog of incoming connection requests
+ :return: None
+ """
+ ret = cm.rdma_listen(self.id, backlog)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Listen Failed')
+
+ def connect(self, ConnParam param=None):
+ """
+ Initiates an active connection request to a remote destination.
+ :param param: Optional connection parameters
+ :return: None
+ """
+ cdef cm.rdma_conn_param *conn = ¶m.conn_param if param else NULL
+ ret = cm.rdma_connect(self.id, conn)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to Connect')
+
+ def disconnect(self):
+ """
+ Disconnects a connection and transitions any associated QP to error
+ state.
+ :return: None
+ """
+ ret = cm.rdma_disconnect(self.id)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to Disconnect')
+
+ def accept(self, ConnParam param=None):
+ """
+ Is called from the listening side to accept a connection or datagram
+ service lookup request.
+ :param param: Optional connection parameters
+ :return: None
+ """
+ cdef cm.rdma_conn_param *conn = ¶m.conn_param if param else NULL
+ ret = cm.rdma_accept(self.id, conn)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to Accept Connection')
+
+ def post_recv(self, MR mr not None):
+ """
+ Posts a recv_wr via QP associated with CMID.
+ Context param of rdma_post_recv C function currently not supported.
+ :param mr: A valid MR object.
+ :return: None
+ """
+ ret = cm.rdma_post_recv(self.id, NULL, mr.buf, mr.mr.length, mr.mr)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to Post Receive')
+
+ def post_send(self, MR mr not None, flags=v.IBV_SEND_SIGNALED):
+ """
+ Posts a message via QP associated with CMID.
+ Context param of rdma_post_send C function currently not supported.
+ :param mr: A valid MR object which contains message to send.
+ :param flags: flags for send work request.
+ :return: None
+ """
+ ret = cm.rdma_post_send(self.id, NULL, mr.buf, mr.mr.length, mr.mr,
+ flags)
+ if ret != 0:
+ raise PyverbsRDMAErrno('Failed to Post Send')
+
+ def get_recv_comp(self):
+ """
+ Polls the receive CQ associated with CMID for a work completion.
+ :return: The retrieved WC or None if there is no completions
+ """
+ cdef v.ibv_wc wc
+ ret = cm.rdma_get_recv_comp(self.id, &wc)
+ if ret < 0:
+ raise PyverbsRDMAErrno('Failed to retrieve receive completion')
+ elif ret == 0:
+ return None
+ return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode,
+ vendor_err=wc.vendor_err, byte_len=wc.byte_len,
+ qp_num=wc.qp_num, src_qp=wc.src_qp,
+ imm_data=wc.imm_data, wc_flags=wc.wc_flags,
+ pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl,
+ dlid_path_bits=wc.dlid_path_bits)
+
+ def get_send_comp(self):
+ """
+ Polls the send CQ associated with CMID for a work completion.
+ :return: The retrieved WC or None if there is no completions
+ """
+ cdef v.ibv_wc wc
+ ret = cm.rdma_get_send_comp(self.id, &wc)
+ if ret < 0:
+ raise PyverbsRDMAErrno('Failed to retrieve send completion')
+ elif ret == 0:
+ return None
+ return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode,
+ vendor_err=wc.vendor_err, byte_len=wc.byte_len,
+ qp_num=wc.qp_num, src_qp=wc.src_qp,
+ imm_data=wc.imm_data, wc_flags=wc.wc_flags,
+ pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl,
+ dlid_path_bits=wc.dlid_path_bits)
diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx
index 58a2aca27fcc..c747822b3b32 100755
--- a/pyverbs/device.pyx
+++ b/pyverbs/device.pyx
@@ -14,6 +14,7 @@ from .pyverbs_error import PyverbsUserError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
cimport pyverbs.libibverbs as v
+from pyverbs.cmid cimport CMID
from pyverbs.xrcd cimport XRCD
from pyverbs.addr cimport GID
from pyverbs.mr import DMMR
@@ -79,16 +80,22 @@ cdef class Context(PyverbsCM):
Initializes a Context object. The function searches the IB devices list
for a device with the name provided by the user. If such a device is
found, it is opened (unless provider attributes were given).
+ In case of cmid argument, CMID object already holds an ibv_context
+ initiated pointer, hence all we have to do is assign this pointer to
+ Context's object pointer.
:param kwargs: Arguments:
* *name* (str)
The RDMA device's name
* *attr* (object)
Device-specific attributes, meaning that the device is to be
opened by the provider
+ * *cmid* (CMID)
+ A CMID object (represents rdma_cm_id struct)
:return: None
"""
cdef int count
cdef v.ibv_device **dev_list
+ cdef CMID cmid
self.pds = weakref.WeakSet()
self.dms = weakref.WeakSet()
@@ -99,7 +106,13 @@ cdef class Context(PyverbsCM):
dev_name = kwargs.get('name')
provider_attr = kwargs.get('attr')
- if dev_name is not None:
+ cmid = kwargs.get('cmid')
+
+ if cmid is not None:
+ self.context = cmid.id.verbs
+ cmid.ctx = self
+ return
+ elif dev_name is not None:
self.name = dev_name
else:
raise PyverbsUserError('Device name must be provided')
diff --git a/pyverbs/librdmacm.pxd b/pyverbs/librdmacm.pxd
new file mode 100755
index 000000000000..935a4ae24e87
--- /dev/null
+++ b/pyverbs/librdmacm.pxd
@@ -0,0 +1,106 @@
+include 'libibverbs.pxd'
+include 'librdmacm_enums.pxd'
+from libc.stdint cimport uint8_t, uint32_t
+
+cdef extern from '<rdma/rdma_cma.h>':
+
+ cdef struct rdma_cm_id:
+ ibv_context *verbs
+ rdma_event_channel *channel
+ void *context
+ ibv_qp *qp
+ rdma_port_space ps
+ uint8_t port_num
+ rdma_cm_event *event
+ ibv_comp_channel *send_cq_channel
+ ibv_cq *send_cq
+ ibv_comp_channel *recv_cq_channel
+ ibv_cq *recv_cq
+ ibv_srq *srq
+ ibv_pd *pd
+ ibv_qp_type qp_type
+
+ cdef struct rdma_event_channel:
+ int fd
+
+ cdef struct rdma_conn_param:
+ const void *private_data
+ uint8_t private_data_len
+ uint8_t responder_resources
+ uint8_t initiator_depth
+ uint8_t flow_control
+ uint8_t retry_count
+ uint8_t rnr_retry_count
+ uint8_t srq
+ uint32_t qp_num
+
+ cdef struct rdma_ud_param:
+ const void *private_data
+ uint8_t private_data_len
+ ibv_ah_attr ah_attr
+ uint32_t qp_num
+ uint32_t qkey
+
+ cdef union param:
+ rdma_conn_param conn
+ rdma_ud_param ud
+
+ cdef struct rdma_cm_event:
+ rdma_cm_id *id
+ rdma_cm_id *listen_id
+ rdma_cm_event_type event
+ int status
+ param param
+
+ cdef struct rdma_addrinfo:
+ int ai_flags
+ int ai_family
+ int ai_qp_type
+ int ai_port_space
+ int ai_src_len
+ int ai_dst_len
+ sockaddr *ai_src_addr
+ sockaddr *ai_dst_addr
+ char *ai_src_canonname
+ char *ai_dst_canonname
+ size_t ai_route_len
+ void *ai_route
+ size_t ai_connect_len
+ void *ai_connect
+ rdma_addrinfo *ai_next
+
+# These non rdmacm structs defined in one of rdma_cma.h's included header files
+ cdef struct sockaddr:
+ unsigned short sa_family
+ char sa_data[14]
+
+ cdef struct in_addr:
+ uint32_t s_addr
+
+ cdef struct sockaddr_in:
+ short sin_family
+ unsigned short sin_port
+ in_addr sin_addr
+ char sin_zero[8]
+
+ int rdma_create_ep(rdma_cm_id **id, rdma_addrinfo *res,
+ ibv_pd *pd, ibv_qp_init_attr *qp_init_attr)
+ void rdma_destroy_ep(rdma_cm_id *id)
+ int rdma_get_request(rdma_cm_id *listen, rdma_cm_id **id)
+ int rdma_connect(rdma_cm_id *id, rdma_conn_param *conn_param)
+ int rdma_disconnect(rdma_cm_id *id)
+ int rdma_listen(rdma_cm_id *id, int backlog)
+ int rdma_accept(rdma_cm_id *id, rdma_conn_param *conn_param)
+ int rdma_getaddrinfo(char *node, char *service, rdma_addrinfo *hints,
+ rdma_addrinfo **res)
+ void rdma_freeaddrinfo(rdma_addrinfo *res)
+
+cdef extern from '<rdma/rdma_verbs.h>':
+ int rdma_post_recv(rdma_cm_id *id, void *context, void *addr,
+ size_t length, ibv_mr *mr)
+ int rdma_post_send(rdma_cm_id *id, void *context, void *addr,
+ size_t length, ibv_mr *mr, int flags)
+ int rdma_get_send_comp(rdma_cm_id *id, ibv_wc *wc)
+ int rdma_get_recv_comp(rdma_cm_id *id, ibv_wc *wc)
+ ibv_mr *rdma_reg_msgs(rdma_cm_id *id, void *addr, size_t length)
+ int rdma_dereg_mr(ibv_mr *mr)
diff --git a/pyverbs/librdmacm_enums.pxd b/pyverbs/librdmacm_enums.pxd
new file mode 100755
index 000000000000..22a3648fb4a6
--- /dev/null
+++ b/pyverbs/librdmacm_enums.pxd
@@ -0,0 +1,32 @@
+cdef extern from '<rdma/rdma_cma.h>':
+
+ cpdef enum rdma_cm_event_type:
+ RDMA_CM_EVENT_ADDR_RESOLVED
+ RDMA_CM_EVENT_ADDR_ERROR
+ RDMA_CM_EVENT_ROUTE_RESOLVED
+ RDMA_CM_EVENT_ROUTE_ERROR
+ RDMA_CM_EVENT_CONNECT_REQUEST
+ RDMA_CM_EVENT_CONNECT_RESPONSE
+ RDMA_CM_EVENT_CONNECT_ERROR
+ RDMA_CM_EVENT_UNREACHABLE
+ RDMA_CM_EVENT_REJECTED
+ RDMA_CM_EVENT_ESTABLISHED
+ RDMA_CM_EVENT_DISCONNECTED
+ RDMA_CM_EVENT_DEVICE_REMOVAL
+ RDMA_CM_EVENT_MULTICAST_JOIN
+ RDMA_CM_EVENT_MULTICAST_ERROR
+ RDMA_CM_EVENT_ADDR_CHANGE
+ RDMA_CM_EVENT_TIMEWAIT_EXIT
+
+ cpdef enum rdma_port_space:
+ RDMA_PS_IPOIB
+ RDMA_PS_TCP
+ RDMA_PS_UDP
+ RDMA_PS_IB
+
+ # Hint flags which control the operation.
+ cpdef enum:
+ RAI_PASSIVE
+ RAI_NUMERICHOST
+ RAI_NOROUTE
+ RAI_FAMILY
diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx
old mode 100644
new mode 100755
index 46cbb36009ce..d6af58f25980
--- a/pyverbs/pd.pyx
+++ b/pyverbs/pd.pyx
@@ -2,9 +2,10 @@
# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
import weakref
-from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
from pyverbs.device cimport Context, DM
+from pyverbs.cmid cimport CMID
from .mr cimport MR, MW, DMMR
from pyverbs.srq cimport SRQ
from pyverbs.addr cimport AH
@@ -15,18 +16,27 @@ cdef extern from 'errno.h':
cdef class PD(PyverbsCM):
- def __cinit__(self, Context context not None):
+ def __cinit__(self, object creator not None):
"""
Initializes a PD object. A reference for the creating Context is kept
so that Python's GC will destroy the objects in the right order.
:param context: The Context object creating the PD
:return: The newly created PD on success
"""
- self.pd = v.ibv_alloc_pd(<v.ibv_context*>context.context)
- if self.pd == NULL:
- raise PyverbsRDMAErrno('Failed to allocate PD', errno)
- self.ctx = context
- context.add_ref(self)
+ if issubclass(type(creator), Context):
+ self.pd = v.ibv_alloc_pd((<Context>creator).context)
+ if self.pd == NULL:
+ raise PyverbsRDMAErrno('Failed to allocate PD')
+ self.ctx = creator
+ elif issubclass(type(creator), CMID):
+ cmid = <CMID>creator
+ self.pd = cmid.id.pd
+ self.ctx = cmid.ctx
+ cmid.pd = self
+ else:
+ raise PyverbsUserError('Cannot create PD from {type}'
+ .format(type=type(creator)))
+ self.ctx.add_ref(self)
self.logger.debug('PD: Allocated ibv_pd')
self.srqs = weakref.WeakSet()
self.mrs = weakref.WeakSet()
--
2.21.0
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH rdma-core 2/5] tests: Fix PD API test
2019-11-04 10:37 [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 1/5] pyverbs: New CMID class Noa Osherovich
@ 2019-11-04 10:37 ` Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 3/5] tests: New CMResources Class Noa Osherovich
` (2 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Noa Osherovich @ 2019-11-04 10:37 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Maxim Chicherin
From: Maxim Chicherin <maximc@mellanox.com>
In order to support PD class in CMID, modifications was made in PD
constructor, as a result one of test_pd's cases broke.
Fix it to create PD properly.
Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
tests/test_pd.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
mode change 100644 => 100755 tests/test_pd.py
diff --git a/tests/test_pd.py b/tests/test_pd.py
old mode 100644
new mode 100755
index 978cf4900146..a8d6eb2fb69f
--- a/tests/test_pd.py
+++ b/tests/test_pd.py
@@ -48,8 +48,7 @@ class PDTest(PyverbsAPITestCase):
try:
PD(None)
except TypeError as te:
- assert 'expected pyverbs.device.Context' in te.args[0]
- assert 'got NoneType' in te.args[0]
+ assert 'must not be None' in te.args[0]
else:
raise PyverbsRDMAErrno('Created a PD with None context')
--
2.21.0
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH rdma-core 3/5] tests: New CMResources Class
2019-11-04 10:37 [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 1/5] pyverbs: New CMID class Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 2/5] tests: Fix PD API test Noa Osherovich
@ 2019-11-04 10:37 ` Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 4/5] tests: Add RDMACM synchronous traffic test Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 5/5] Documentation: Document creation of CMID Noa Osherovich
4 siblings, 0 replies; 8+ messages in thread
From: Noa Osherovich @ 2019-11-04 10:37 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Maxim Chicherin
From: Maxim Chicherin <maximc@mellanox.com>
A base aggregation object for RDMACM. Currently only synchronous data
path flow is supported.
Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
tests/base.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/tests/base.py b/tests/base.py
index e4e92fa2a35f..54bfc42ffe4c 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -11,13 +11,16 @@ from pyverbs.qp import QPCap, QPInitAttrEx, QPInitAttr, QPAttr, QP
from pyverbs.addr import AHAttr, GlobalRoute
from pyverbs.xrcd import XRCD, XRCDInitAttr
from pyverbs.srq import SRQ, SrqInitAttrEx
+from pyverbs.cmid import CMID, AddrInfo
from pyverbs.device import Context
+import pyverbs.cm_enums as ce
import pyverbs.device as d
import pyverbs.enums as e
from pyverbs.pd import PD
from pyverbs.cq import CQ
from pyverbs.mr import MR
+
PATH_MTU = e.IBV_MTU_1024
MAX_DEST_RD_ATOMIC = 1
MAX_RD_ATOMIC = 1
@@ -26,6 +29,10 @@ RETRY_CNT = 7
RNR_RETRY = 7
TIMEOUT = 14
+# for rdmacm
+PORT = '7471'
+ZERO_ADDR = '0.0.0.0'
+
class PyverbsAPITestCase(unittest.TestCase):
def setUp(self):
@@ -144,6 +151,51 @@ class RDMATestCase(unittest.TestCase):
self._add_gids_per_port(ctx, dev, port+1)
+class CMResources:
+ """
+ CMResources class is a base aggregator object which contains basic
+ resources for RDMA CM communication.
+ """
+ def __init__(self, src=ZERO_ADDR, dst=ZERO_ADDR, port=PORT):
+ """
+ :param src: Local address to bind to (for passive side)
+ :param dst: Destination address to connect (for active side)
+ :param port: Port number of the address
+ """
+ self.is_server = True if dst == ZERO_ADDR else False
+ self.qp_init_attr = None
+ self.msg_size = 1024
+ self.num_msgs = 100
+ self.new_id = None
+ self.port = port
+ self.mr = None
+ if self.is_server:
+ self.ai = AddrInfo(src, self.port, ce.RDMA_PS_TCP,
+ ce.RAI_PASSIVE)
+ else:
+ self.ai = AddrInfo(dst, self.port, ce.RDMA_PS_TCP)
+ self.create_qp_init_attr()
+ self.cmid = CMID(creator=self.ai, qp_init_attr=self.qp_init_attr)
+
+ def create_mr(self, cmid):
+ self.mr = cmid.reg_msgs(self.msg_size)
+
+ def create_qp_init_attr(self):
+ self.qp_init_attr = QPInitAttr(cap=QPCap(max_recv_wr=1))
+
+ def listen(self):
+ self.cmid.listen()
+ self.new_id = self.cmid.get_request()
+ self.new_id.accept()
+
+ def pre_run(self):
+ if self.is_server:
+ self.listen()
+ else:
+ self.cmid.connect()
+ self.create_mr(self.new_id if self.is_server else self.cmid)
+
+
class BaseResources(object):
"""
BaseResources class is a base aggregator object which contains basic
--
2.21.0
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH rdma-core 4/5] tests: Add RDMACM synchronous traffic test
2019-11-04 10:37 [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs Noa Osherovich
` (2 preceding siblings ...)
2019-11-04 10:37 ` [PATCH rdma-core 3/5] tests: New CMResources Class Noa Osherovich
@ 2019-11-04 10:37 ` Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 5/5] Documentation: Document creation of CMID Noa Osherovich
4 siblings, 0 replies; 8+ messages in thread
From: Noa Osherovich @ 2019-11-04 10:37 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Maxim Chicherin
From: Maxim Chicherin <maximc@mellanox.com>
This test creates active and passive sides for RDMACM communication
and validates received messages.
Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
tests/CMakeLists.txt | 2 ++
tests/rdmacm_utils.py | 43 ++++++++++++++++++++++++++++++++
tests/test_rdmacm.py | 57 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 102 insertions(+)
mode change 100644 => 100755 tests/CMakeLists.txt
create mode 100755 tests/rdmacm_utils.py
create mode 100755 tests/test_rdmacm.py
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
old mode 100644
new mode 100755
index 960276230860..0d81d1a98fb7
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -3,6 +3,7 @@
rdma_python_test(tests
__init__.py
+ rdmacm_utils.py
test_addr.py
base.py
test_cq.py
@@ -11,6 +12,7 @@ rdma_python_test(tests
test_pd.py
test_qp.py
test_odp.py
+ test_rdmacm.py
utils.py
)
diff --git a/tests/rdmacm_utils.py b/tests/rdmacm_utils.py
new file mode 100755
index 000000000000..59b627393e4f
--- /dev/null
+++ b/tests/rdmacm_utils.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
+"""
+Provide some useful helper function for pyverbs rdmacm' tests.
+"""
+from tests.base import CMResources
+from tests.utils import validate
+
+
+def active_side(node):
+ client = CMResources(dst=node)
+ client.pre_run()
+ connected_id = client.cmid
+ send_msg = 'c' * client.msg_size
+ for _ in range(client.num_msgs):
+ client.mr.write(send_msg, client.msg_size)
+ connected_id.post_send(client.mr)
+ while connected_id.get_send_comp() is None:
+ pass
+ connected_id.post_recv(client.mr)
+ while connected_id.get_recv_comp() is None:
+ pass
+ msg_received = client.mr.read(client.msg_size, 0)
+ validate(msg_received, False, client.msg_size)
+ connected_id.disconnect()
+
+
+def passive_side(node):
+ server = CMResources(src=node)
+ server.pre_run()
+ connected_id = server.new_id
+ send_msg = 's' * server.msg_size
+ for _ in range(server.num_msgs):
+ connected_id.post_recv(server.mr)
+ while connected_id.get_recv_comp() is None:
+ pass
+ msg_received = server.mr.read(server.msg_size, 0)
+ validate(msg_received, True, server.msg_size)
+ server.mr.write(send_msg, server.msg_size)
+ connected_id.post_send(server.mr)
+ while connected_id.get_send_comp() is None:
+ pass
+ connected_id.disconnect()
diff --git a/tests/test_rdmacm.py b/tests/test_rdmacm.py
new file mode 100755
index 000000000000..e435c635d7b2
--- /dev/null
+++ b/tests/test_rdmacm.py
@@ -0,0 +1,57 @@
+from tests.rdmacm_utils import active_side, passive_side
+from tests.base import RDMATestCase
+from multiprocessing import Process
+import pyverbs.device as d
+import subprocess
+import unittest
+import json
+
+
+class CMTestCase(RDMATestCase):
+ def setUp(self):
+ if self.dev_name is not None:
+ net_name = self.get_net_name(self.dev_name)
+ try:
+ self.ip_addr = self.get_ip_address(net_name)
+ except KeyError:
+ raise unittest.SkipTest('Device {} has no net interface'
+ .format(self.dev_name))
+ else:
+ dev_list = d.get_device_list()
+ for dev in dev_list:
+ net_name = self.get_net_name(dev.name.decode())
+ try:
+ self.ip_addr = self.get_ip_address(net_name)
+ except IndexError:
+ continue
+ else:
+ self.dev_name = dev.name.decode()
+ break
+ if self.dev_name is None:
+ raise unittest.SkipTest('No devices with net interface')
+ super().setUp()
+
+ @staticmethod
+ def get_net_name(dev):
+ process = subprocess.Popen(['ls', '/sys/class/infiniband/{}/device/net/'
+ .format(dev)], stdout=subprocess.PIPE)
+ out, err = process.communicate()
+ return out.decode().split('\n')[0]
+
+ @staticmethod
+ def get_ip_address(ifname):
+ process = subprocess.Popen(['ip', '-j', 'addr', 'show', ifname],
+ stdout=subprocess.PIPE)
+ out, err = process.communicate()
+ loaded_json = json.loads(out.decode())
+ interface = loaded_json[0]['addr_info'][0]['local']
+ if 'fe80::' in interface:
+ interface = interface + '%' + ifname
+ return interface
+
+ def test_rdmacm_sync_traffic(self):
+ ps = Process(target=passive_side, args=[self.ip_addr])
+ ps.start()
+ ps.join(1)
+ active_side(self.ip_addr)
+ ps.join()
--
2.21.0
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH rdma-core 5/5] Documentation: Document creation of CMID
2019-11-04 10:37 [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs Noa Osherovich
` (3 preceding siblings ...)
2019-11-04 10:37 ` [PATCH rdma-core 4/5] tests: Add RDMACM synchronous traffic test Noa Osherovich
@ 2019-11-04 10:37 ` Noa Osherovich
4 siblings, 0 replies; 8+ messages in thread
From: Noa Osherovich @ 2019-11-04 10:37 UTC (permalink / raw)
To: dledford, Jason Gunthorpe, Leon Romanovsky; +Cc: linux-rdma, Maxim Chicherin
From: Maxim Chicherin <maximc@mellanox.com>
Add code snippet to demonstrate creation of CMID and establish
connection (synchronous control path).
Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
Documentation/pyverbs.md | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md
index 29ab9592c53c..6ff6fc5bbc37 100755
--- a/Documentation/pyverbs.md
+++ b/Documentation/pyverbs.md
@@ -390,3 +390,33 @@ srq_attr.comp_mask = e.IBV_SRQ_INIT_ATTR_TYPE | e.IBV_SRQ_INIT_ATTR_PD | \
e.IBV_SRQ_INIT_ATTR_CQ | e.IBV_SRQ_INIT_ATTR_XRCD
srq = SRQ(ctx, srq_attr)
```
+
+##### CMID
+The following code snippet will demonstrate creation of a CMID object, which
+represents rdma_cm_id C struct, and establish connection between two peers.
+Currently only synchronous control path is supported (rdma_create_ep).
+For more complex examples, please see tests/test_rdmacm.
+```python
+from pyverbs.qp import QPInitAttr, QPCap
+from pyverbs.cmid import CMID, AddrInfo
+import pyverbs.cm_enums as ce
+
+cap = QPCap(max_recv_wr=1)
+qp_init_attr = QPInitAttr(cap=cap)
+server = '11.137.14.124'
+port = '7471'
+
+# Active side
+
+cai = AddrInfo(server, port, ce.RDMA_PS_TCP)
+cid = CMID(creator=cai, qp_init_attr=qp_init_attr)
+cid.connect() # send connection request to server
+
+# Passive side
+
+sai = AddrInfo(server, port, ce.RDMA_PS_TCP, ce.RAI_PASSIVE)
+sid = CMID(creator=sai, qp_init_attr=qp_init_attr)
+sid.listen() # listen for incoming connection requests
+new_id = sid.get_request() # check if there are any connection requests
+new_id.accept() # new_id is connected to remote peer and ready to communicate
+```
--
2.21.0
^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH rdma-core 1/5] pyverbs: New CMID class
2019-11-04 10:37 ` [PATCH rdma-core 1/5] pyverbs: New CMID class Noa Osherovich
@ 2019-11-04 11:09 ` Benjamin Drung
2019-11-05 14:46 ` Leon Romanovsky
0 siblings, 1 reply; 8+ messages in thread
From: Benjamin Drung @ 2019-11-04 11:09 UTC (permalink / raw)
To: Noa Osherovich, dledford, Jason Gunthorpe, Leon Romanovsky
Cc: linux-rdma, Maxim Chicherin
Am Montag, den 04.11.2019, 10:37 +0000 schrieb Noa Osherovich:
> diff --git a/pyverbs/cm_enums.pyx b/pyverbs/cm_enums.pyx
> new file mode 120000
> index 000000000000..bdab2b585a1d
> --- /dev/null
> +++ b/pyverbs/cm_enums.pyx
> @@ -0,0 +1 @@
> +librdmacm_enums.pxd
> \ No newline at end of file
> diff --git a/pyverbs/cmid.pxd b/pyverbs/cmid.pxd
> new file mode 100755
> index 000000000000..56bc755daf42
> --- /dev/null
> +++ b/pyverbs/cmid.pxd
> @@ -0,0 +1,25 @@
> +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> +# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
> See COPYING file
> +
> +#cython: language_level=3
> +
> +from pyverbs.base cimport PyverbsObject, PyverbsCM
> +from libc.string cimport memcpy, memset
> +from libc.stdlib cimport free, malloc
> +cimport pyverbs.librdmacm as cm
> +
> +
> +cdef class CMID(PyverbsCM):
> + cdef cm.rdma_cm_id *id
> + cdef object ctx
> + cdef object pd
> + cpdef close(self)
> +
> +
> +cdef class AddrInfo(PyverbsObject):
> + cdef cm.rdma_addrinfo *addr_info
> + cpdef close(self)
> +
> +
> +cdef class ConnParam(PyverbsObject):
> + cdef cm.rdma_conn_param conn_param
> \ No newline at end of file
Please add newlines to the end of these files.
--
Benjamin Drung
Debian & Ubuntu Developer
Platform Engineering Compute (Enterprise Cloud)
1&1 IONOS SE | Greifswalder Str. 207 | 10405 Berlin | Germany
E-mail: benjamin.drung@cloud.ionos.com | Web: www.ionos.de
Hauptsitz Montabaur, Amtsgericht Montabaur, HRB 24498
Vorstand: Dr. Christian Böing, Hüseyin Dogan, Hans-Henning Kettler,
Matthias Steinberg, Achim Weiß
Aufsichtsratsvorsitzender: Markus Kadelke
Member of United Internet
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH rdma-core 1/5] pyverbs: New CMID class
2019-11-04 11:09 ` Benjamin Drung
@ 2019-11-05 14:46 ` Leon Romanovsky
0 siblings, 0 replies; 8+ messages in thread
From: Leon Romanovsky @ 2019-11-05 14:46 UTC (permalink / raw)
To: Benjamin Drung
Cc: Noa Osherovich, dledford, Jason Gunthorpe, linux-rdma, Maxim Chicherin
On Mon, Nov 04, 2019 at 12:09:59PM +0100, Benjamin Drung wrote:
> Am Montag, den 04.11.2019, 10:37 +0000 schrieb Noa Osherovich:
> > diff --git a/pyverbs/cm_enums.pyx b/pyverbs/cm_enums.pyx
> > new file mode 120000
> > index 000000000000..bdab2b585a1d
> > --- /dev/null
> > +++ b/pyverbs/cm_enums.pyx
> > @@ -0,0 +1 @@
> > +librdmacm_enums.pxd
> > \ No newline at end of file
> > diff --git a/pyverbs/cmid.pxd b/pyverbs/cmid.pxd
> > new file mode 100755
> > index 000000000000..56bc755daf42
> > --- /dev/null
> > +++ b/pyverbs/cmid.pxd
> > @@ -0,0 +1,25 @@
> > +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> > +# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
> > See COPYING file
> > +
> > +#cython: language_level=3
> > +
> > +from pyverbs.base cimport PyverbsObject, PyverbsCM
> > +from libc.string cimport memcpy, memset
> > +from libc.stdlib cimport free, malloc
> > +cimport pyverbs.librdmacm as cm
> > +
> > +
> > +cdef class CMID(PyverbsCM):
> > + cdef cm.rdma_cm_id *id
> > + cdef object ctx
> > + cdef object pd
> > + cpdef close(self)
> > +
> > +
> > +cdef class AddrInfo(PyverbsObject):
> > + cdef cm.rdma_addrinfo *addr_info
> > + cpdef close(self)
> > +
> > +
> > +cdef class ConnParam(PyverbsObject):
> > + cdef cm.rdma_conn_param conn_param
> > \ No newline at end of file
>
> Please add newlines to the end of these files.
PR was updated.
Thanks
>
> --
> Benjamin Drung
>
> Debian & Ubuntu Developer
> Platform Engineering Compute (Enterprise Cloud)
>
> 1&1 IONOS SE | Greifswalder Str. 207 | 10405 Berlin | Germany
> E-mail: benjamin.drung@cloud.ionos.com | Web: www.ionos.de
>
> Hauptsitz Montabaur, Amtsgericht Montabaur, HRB 24498
> Vorstand: Dr. Christian Böing, Hüseyin Dogan, Hans-Henning Kettler,
> Matthias Steinberg, Achim Weiß
> Aufsichtsratsvorsitzender: Markus Kadelke
> Member of United Internet
>
^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2019-11-05 14:46 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-11-04 10:37 [PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 1/5] pyverbs: New CMID class Noa Osherovich
2019-11-04 11:09 ` Benjamin Drung
2019-11-05 14:46 ` Leon Romanovsky
2019-11-04 10:37 ` [PATCH rdma-core 2/5] tests: Fix PD API test Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 3/5] tests: New CMResources Class Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 4/5] tests: Add RDMACM synchronous traffic test Noa Osherovich
2019-11-04 10:37 ` [PATCH rdma-core 5/5] Documentation: Document creation of CMID Noa Osherovich
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).