[PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs

[PATCH rdma-core 0/5] Introducing RDMACM support in pyverbs

[Noa Osherovich]

The following patches add support for RDMACM in pyverbs. Currently
only synchronous data path is supported (creation using rdma_create_ep).
Testing infrastructure for RDMACM is also added as well as a synchronous
traffic test.


Maxim Chicherin (5):
  pyverbs: New CMID class
  tests: Fix PD API test
  tests: New CMResources Class
  tests: Add RDMACM synchronous traffic test
  Documentation: Document creation of CMID

 Documentation/pyverbs.md         |  30 ++++
 buildlib/pyverbs_functions.cmake |   2 +-
 pyverbs/CMakeLists.txt           |   2 +
 pyverbs/cm_enums.pyx             |   1 +
 pyverbs/cmid.pxd                 |  25 +++
 pyverbs/cmid.pyx                 | 285 +++++++++++++++++++++++++++++++
 pyverbs/device.pyx               |  15 +-
 pyverbs/librdmacm.pxd            | 106 ++++++++++++
 pyverbs/librdmacm_enums.pxd      |  32 ++++
 pyverbs/pd.pyx                   |  24 ++-
 tests/CMakeLists.txt             |   2 +
 tests/base.py                    |  52 ++++++
 tests/rdmacm_utils.py            |  43 +++++
 tests/test_pd.py                 |   3 +-
 tests/test_rdmacm.py             |  57 +++++++
 15 files changed, 668 insertions(+), 11 deletions(-)
 create mode 120000 pyverbs/cm_enums.pyx
 create mode 100755 pyverbs/cmid.pxd
 create mode 100755 pyverbs/cmid.pyx
 create mode 100755 pyverbs/librdmacm.pxd
 create mode 100755 pyverbs/librdmacm_enums.pxd
 mode change 100644 => 100755 pyverbs/pd.pyx
 mode change 100644 => 100755 tests/CMakeLists.txt
 create mode 100755 tests/rdmacm_utils.py
 mode change 100644 => 100755 tests/test_pd.py
 create mode 100755 tests/test_rdmacm.py

-- 
2.21.0

[PATCH rdma-core 1/5] pyverbs: New CMID class

[Noa Osherovich]

From: Maxim Chicherin <maximc@mellanox.com>

The CMID class represents rdma_cm_id, RDMA Connection Manager.
Currently only synchronous data path is supported. Support was added
to Context and PD classes to allow creation using rdmacm's API.

Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
 buildlib/pyverbs_functions.cmake |   2 +-
 pyverbs/CMakeLists.txt           |   2 +
 pyverbs/cm_enums.pyx             |   1 +
 pyverbs/cmid.pxd                 |  25 +++
 pyverbs/cmid.pyx                 | 285 +++++++++++++++++++++++++++++++
 pyverbs/device.pyx               |  15 +-
 pyverbs/librdmacm.pxd            | 106 ++++++++++++
 pyverbs/librdmacm_enums.pxd      |  32 ++++
 pyverbs/pd.pyx                   |  24 ++-
 9 files changed, 483 insertions(+), 9 deletions(-)
 create mode 120000 pyverbs/cm_enums.pyx
 create mode 100755 pyverbs/cmid.pxd
 create mode 100755 pyverbs/cmid.pyx
 create mode 100755 pyverbs/librdmacm.pxd
 create mode 100755 pyverbs/librdmacm_enums.pxd
 mode change 100644 => 100755 pyverbs/pd.pyx

diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake
index 4c255054fe94..a494ec16610b 100644
--- a/buildlib/pyverbs_functions.cmake
+++ b/buildlib/pyverbs_functions.cmake
@@ -25,7 +25,7 @@ function(rdma_cython_module PY_MODULE LINKER_FLAGS)
       COMPILE_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option ${NO_VAR_TRACKING_FLAGS}"
       LIBRARY_OUTPUT_DIRECTORY "${BUILD_PYTHON}/${PY_MODULE}"
       PREFIX "")
-    target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs ${LINKER_FLAGS})
+    target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs rdmacm ${LINKER_FLAGS})
     install(TARGETS ${SONAME}
       DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
   endforeach()
diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt
index 7bbb5fc841c0..de37025ce324 100755
--- a/pyverbs/CMakeLists.txt
+++ b/pyverbs/CMakeLists.txt
@@ -4,6 +4,8 @@
 rdma_cython_module(pyverbs ""
   addr.pyx
   base.pyx
+  cm_enums.pyx
+  cmid.pyx
   cq.pyx
   device.pyx
   enums.pyx
diff --git a/pyverbs/cm_enums.pyx b/pyverbs/cm_enums.pyx
new file mode 120000
index 000000000000..bdab2b585a1d
--- /dev/null
+++ b/pyverbs/cm_enums.pyx
@@ -0,0 +1 @@
+librdmacm_enums.pxd
\ No newline at end of file
diff --git a/pyverbs/cmid.pxd b/pyverbs/cmid.pxd
new file mode 100755
index 000000000000..56bc755daf42
--- /dev/null
+++ b/pyverbs/cmid.pxd
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from pyverbs.base cimport PyverbsObject, PyverbsCM
+from libc.string cimport memcpy, memset
+from libc.stdlib cimport free, malloc
+cimport pyverbs.librdmacm as cm
+
+
+cdef class CMID(PyverbsCM):
+    cdef cm.rdma_cm_id *id
+    cdef object ctx
+    cdef object pd
+    cpdef close(self)
+
+
+cdef class AddrInfo(PyverbsObject):
+    cdef cm.rdma_addrinfo *addr_info
+    cpdef close(self)
+
+
+cdef class ConnParam(PyverbsObject):
+    cdef cm.rdma_conn_param conn_param
\ No newline at end of file
diff --git a/pyverbs/cmid.pyx b/pyverbs/cmid.pyx
new file mode 100755
index 000000000000..c752feda8781
--- /dev/null
+++ b/pyverbs/cmid.pyx
@@ -0,0 +1,285 @@
+from pyverbs.pyverbs_error import PyverbsError
+from pyverbs.device cimport PortAttr, Context
+from pyverbs.qp cimport QPInitAttr, QPAttr
+from pyverbs.base import PyverbsRDMAErrno
+cimport pyverbs.libibverbs_enums as e
+cimport pyverbs.librdmacm_enums as ce
+cimport pyverbs.libibverbs as v
+cimport pyverbs.librdmacm as cm
+from pyverbs.pd cimport PD
+from pyverbs.mr cimport MR
+from pyverbs.cq cimport WC
+
+
+cdef class ConnParam(PyverbsObject):
+
+    def __cinit__(self, resources=1, depth=1, flow_control=0, retry=5,
+                  rnr_retry=5, srq=0, qp_num=0):
+        """
+        Initialize a ConnParam object over an underlying rdma_conn_param
+        C object which contains connection parameters. There are a few types of
+        port spaces in RDMACM: RDMA_PS_TCP, RDMA_PS_UDP, RDMA_PS_IB and
+        RDMA_PS_IPOIB. RDMA_PS_TCP resembles RC QP connection, which provides
+        reliable, connection-oriented QP communication. This object applies only
+        to RDMA_PS_TCP port space.
+        :param resources: Max outstanding RDMA read and atomic ops that local
+                          side will accept from the remote side.
+        :param depth: Max outstanding RDMA read and atomic ops that local side
+                      will have to the remote side.
+        :param flow_control: Specifies if hardware flow control is available.
+        :param retry: Max number of times that a send, RDMA or atomic op from
+                      the remote peer should be retried.
+        :param rnr_retry: The maximum number of times that a send operation from
+                          the remote peer should be retried on a connection
+                          after receiving a receiver not ready (RNR) error.
+        :param srq: Specifies if the QP using shared receive queue, ignored if
+                    the QP created by CMID.
+        :param qp_num: Specifies the QP number, ignored if the QP created by
+                       CMID.
+        :return: ConnParam object
+        """
+        memset(&self.conn_param, 0, sizeof(cm.rdma_conn_param))
+        self.conn_param.responder_resources = resources
+        self.conn_param.initiator_depth = depth
+        self.conn_param.flow_control = flow_control
+        self.conn_param.retry_count = retry
+        self.conn_param.rnr_retry_count = rnr_retry
+        self.conn_param.srq = srq
+        self.conn_param.qp_num = qp_num
+
+    def __str__(self):
+        print_format  = '{:<4}: {:<4}\n'
+        return '{}: {}\n'.format('Connection parameters', "") +\
+               print_format.format('responder resources', self.conn_param.responder_resources) +\
+               print_format.format('initiator depth', self.conn_param.initiator_depth) +\
+               print_format.format('flow control', self.conn_param.flow_control) +\
+               print_format.format('retry count', self.conn_param.retry_count) +\
+               print_format.format('rnr retry count', self.conn_param.rnr_retry_count) +\
+               print_format.format('srq', self.conn_param.srq) +\
+               print_format.format('qp number', self.conn_param.qp_num)
+
+
+cdef class AddrInfo(PyverbsObject):
+    def __cinit__(self, node=None, service=None, port_space=0, flags=0):
+        """
+        Initialize an AddrInfo object over an underlying rdma_addrinfo C object.
+        :param node: Name, dotted-decimal IPv4 or IPv6 hex address to resolve.
+        :param service: The service name or port number of the address.
+        :param port_space: RDMA port space used (RDMA_PS_UDP or RDMA_PS_TCP).
+        :param flags: Hint flags which control the operation.
+        :return: An AddrInfo object which contains information needed to
+        establish communication.
+        """
+        cdef char* srvc = NULL
+        cdef char* address = NULL
+        cdef cm.rdma_addrinfo hints
+        cdef cm.rdma_addrinfo *hints_ptr = NULL
+
+        if node is not None:
+            node = node.encode('utf-8')
+            address = <char*>node
+        if service is not None:
+            service = service.encode('utf-8')
+            srvc = <char*>service
+        if port_space != 0:
+            hints_ptr = &hints
+            memset(hints_ptr, 0, sizeof(cm.rdma_addrinfo))
+            hints.ai_port_space = port_space
+            hints.ai_flags = flags
+        ret = cm.rdma_getaddrinfo(address, srvc, hints_ptr, &self.addr_info)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to get Address Info')
+
+    def __dealloc__(self):
+        self.close()
+
+    cpdef close(self):
+        self.logger.debug('Closing AddrInfo')
+        if self.addr_info != NULL:
+            cm.rdma_freeaddrinfo(self.addr_info)
+        self.addr_info = NULL
+
+
+cdef class CMID(PyverbsCM):
+
+    def __cinit__(self, object creator=None, QPInitAttr qp_init_attr=None,
+                  PD pd=None):
+        """
+        Initialize a CMID object over an underlying rdma_cm_id C object.
+        This is the main RDMA CM object which provides most of the rdmacm API.
+        Currently only synchronous RDMA_PS_TCP communication supported.
+        :param creator: For synchronous communication we need AddrInfo object in
+                        order to establish connection. We allow creator to be
+                        None for inner usage, see get_request method.
+        :param pd: Optional parameter, a PD to be associated with this CMID.
+        :param qp_init_attr: Optional initial QP attributes of CMID
+                             associated QP.
+        :return: CMID object for synchronous communication.
+        """
+        cdef v.ibv_qp_init_attr *init
+        cdef v.ibv_pd *in_pd = NULL
+        self.pd = None
+        self.ctx = None
+        if creator is None:
+            return
+        elif issubclass(type(creator), AddrInfo):
+            init = NULL if qp_init_attr is None else &qp_init_attr.attr
+            if pd is not None:
+                in_pd = pd.pd
+                self.pd = pd
+            ret = cm.rdma_create_ep(&self.id, (<AddrInfo>creator).addr_info,
+                                    in_pd, init)
+            if ret != 0:
+                raise PyverbsRDMAErrno('Failed to create CM ID')
+            if not (<AddrInfo>creator).addr_info.ai_flags & ce.RAI_PASSIVE:
+                self.ctx = Context(cmid=self)
+                if self.pd is None:
+                    self.pd = PD(self)
+        else:
+            raise PyverbsRDMAErrno('Cannot create CM ID from {obj}'
+                                    .format(obj=type(creator)))
+
+    def __dealloc__(self):
+        self.close()
+
+    cpdef close(self):
+        self.logger.debug('Closing CMID')
+        if self.id != NULL:
+            cm.rdma_destroy_ep(self.id)
+            if self.ctx:
+                (<Context>self.ctx).context = NULL
+            if self.pd:
+                (<PD>self.pd).pd = NULL
+            self.id = NULL
+
+    def get_request(self):
+        """
+        Retrieves the next pending connection request event. The call may only
+        be used on listening CMIDs operating synchronously. If the call is
+        successful, a new CMID representing the connection request will be
+        returned to the user. The new CMID will reference event information
+        associated with the request until the user calls reject, accept, or
+        close on the newly created identifier.
+        :return: New CMID representing the connection request.
+        """
+        to_conn = CMID()
+        ret = cm.rdma_get_request(self.id, &to_conn.id)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to get request, no connection established')
+        self.ctx = Context(cmid=to_conn)
+        self.pd = PD(to_conn)
+        return to_conn
+
+    def reg_msgs(self, size):
+        """
+        Registers a memory region for sending or receiving messages or for
+        RDMA operations. The registered memory may then be posted to an CMID
+        using post_send or post_recv methods.
+        :param size: The total length of the memory to register
+        :return: registered MR
+        """
+        return MR(self.pd, size, e.IBV_ACCESS_LOCAL_WRITE)
+
+    def listen(self, backlog=0):
+        """
+        Listen for incoming connection requests or datagram service lookup.
+        The listen is restricted to the locally bound source address.
+        :param backlog: The backlog of incoming connection requests
+        :return: None
+        """
+        ret = cm.rdma_listen(self.id, backlog)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Listen Failed')
+
+    def connect(self, ConnParam param=None):
+        """
+        Initiates an active connection request to a remote destination.
+        :param param: Optional connection parameters
+        :return: None
+        """
+        cdef cm.rdma_conn_param *conn = &param.conn_param if param else NULL
+        ret = cm.rdma_connect(self.id, conn)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to Connect')
+
+    def disconnect(self):
+        """
+        Disconnects a connection and transitions any associated QP to error
+        state.
+        :return: None
+        """
+        ret = cm.rdma_disconnect(self.id)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to Disconnect')
+
+    def accept(self, ConnParam param=None):
+        """
+        Is called from the listening side to accept a connection or datagram
+        service lookup request.
+        :param param: Optional connection parameters
+        :return: None
+        """
+        cdef cm.rdma_conn_param *conn = &param.conn_param if param else NULL
+        ret = cm.rdma_accept(self.id, conn)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to Accept Connection')
+
+    def post_recv(self, MR mr not None):
+        """
+        Posts a recv_wr via QP associated with CMID.
+        Context param of rdma_post_recv C function currently not supported.
+        :param mr: A valid MR object.
+        :return: None
+        """
+        ret = cm.rdma_post_recv(self.id, NULL, mr.buf, mr.mr.length, mr.mr)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to Post Receive')
+
+    def post_send(self, MR mr not None, flags=v.IBV_SEND_SIGNALED):
+        """
+        Posts a message via QP associated with CMID.
+        Context param of rdma_post_send C function currently not supported.
+        :param mr: A valid MR object which contains message to send.
+        :param flags: flags for send work request.
+        :return: None
+        """
+        ret = cm.rdma_post_send(self.id, NULL, mr.buf, mr.mr.length, mr.mr,
+                                flags)
+        if ret != 0:
+            raise PyverbsRDMAErrno('Failed to Post Send')
+
+    def get_recv_comp(self):
+        """
+        Polls the receive CQ associated with CMID for a work completion.
+        :return: The retrieved WC or None if there is no completions
+        """
+        cdef v.ibv_wc wc
+        ret = cm.rdma_get_recv_comp(self.id, &wc)
+        if ret < 0:
+            raise PyverbsRDMAErrno('Failed to retrieve receive completion')
+        elif ret == 0:
+            return None
+        return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode,
+                  vendor_err=wc.vendor_err, byte_len=wc.byte_len,
+                  qp_num=wc.qp_num, src_qp=wc.src_qp,
+                  imm_data=wc.imm_data, wc_flags=wc.wc_flags,
+                  pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl,
+                  dlid_path_bits=wc.dlid_path_bits)
+
+    def get_send_comp(self):
+        """
+        Polls the send CQ associated with CMID for a work completion.
+        :return: The retrieved WC or None if there is no completions
+        """
+        cdef v.ibv_wc wc
+        ret = cm.rdma_get_send_comp(self.id, &wc)
+        if ret < 0:
+            raise PyverbsRDMAErrno('Failed to retrieve send completion')
+        elif ret == 0:
+            return None
+        return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode,
+                  vendor_err=wc.vendor_err, byte_len=wc.byte_len,
+                  qp_num=wc.qp_num, src_qp=wc.src_qp,
+                  imm_data=wc.imm_data, wc_flags=wc.wc_flags,
+                  pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl,
+                  dlid_path_bits=wc.dlid_path_bits)
diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx
index 58a2aca27fcc..c747822b3b32 100755
--- a/pyverbs/device.pyx
+++ b/pyverbs/device.pyx
@@ -14,6 +14,7 @@ from .pyverbs_error import PyverbsUserError
 from pyverbs.base import PyverbsRDMAErrno
 cimport pyverbs.libibverbs_enums as e
 cimport pyverbs.libibverbs as v
+from pyverbs.cmid cimport CMID
 from pyverbs.xrcd cimport XRCD
 from pyverbs.addr cimport GID
 from pyverbs.mr import DMMR
@@ -79,16 +80,22 @@ cdef class Context(PyverbsCM):
         Initializes a Context object. The function searches the IB devices list
         for a device with the name provided by the user. If such a device is
         found, it is opened (unless provider attributes were given).
+        In case of cmid argument, CMID object already holds an ibv_context
+        initiated pointer, hence all we have to do is assign this pointer to
+        Context's object pointer.
         :param kwargs: Arguments:
             * *name* (str)
                The RDMA device's name
             * *attr* (object)
                Device-specific attributes, meaning that the device is to be
                opened by the provider
+            * *cmid* (CMID)
+                A CMID object (represents rdma_cm_id struct)
         :return: None
         """
         cdef int count
         cdef v.ibv_device **dev_list
+        cdef CMID cmid
 
         self.pds = weakref.WeakSet()
         self.dms = weakref.WeakSet()
@@ -99,7 +106,13 @@ cdef class Context(PyverbsCM):
 
         dev_name = kwargs.get('name')
         provider_attr = kwargs.get('attr')
-        if dev_name is not None:
+        cmid = kwargs.get('cmid')
+
+        if cmid is not None:
+            self.context = cmid.id.verbs
+            cmid.ctx = self
+            return
+        elif dev_name is not None:
             self.name = dev_name
         else:
             raise PyverbsUserError('Device name must be provided')
diff --git a/pyverbs/librdmacm.pxd b/pyverbs/librdmacm.pxd
new file mode 100755
index 000000000000..935a4ae24e87
--- /dev/null
+++ b/pyverbs/librdmacm.pxd
@@ -0,0 +1,106 @@
+include 'libibverbs.pxd'
+include 'librdmacm_enums.pxd'
+from libc.stdint cimport uint8_t, uint32_t
+
+cdef extern from '<rdma/rdma_cma.h>':
+
+    cdef struct rdma_cm_id:
+        ibv_context         *verbs
+        rdma_event_channel  *channel
+        void                *context
+        ibv_qp              *qp
+        rdma_port_space     ps
+        uint8_t             port_num
+        rdma_cm_event       *event
+        ibv_comp_channel    *send_cq_channel
+        ibv_cq              *send_cq
+        ibv_comp_channel    *recv_cq_channel
+        ibv_cq              *recv_cq
+        ibv_srq             *srq
+        ibv_pd              *pd
+        ibv_qp_type         qp_type
+
+    cdef struct rdma_event_channel:
+        int fd
+
+    cdef struct rdma_conn_param:
+        const void *private_data
+        uint8_t     private_data_len
+        uint8_t     responder_resources
+        uint8_t     initiator_depth
+        uint8_t     flow_control
+        uint8_t     retry_count
+        uint8_t     rnr_retry_count
+        uint8_t     srq
+        uint32_t    qp_num
+
+    cdef struct rdma_ud_param:
+        const void  *private_data
+        uint8_t     private_data_len
+        ibv_ah_attr ah_attr
+        uint32_t    qp_num
+        uint32_t    qkey
+
+    cdef union param:
+        rdma_conn_param conn
+        rdma_ud_param   ud
+
+    cdef struct rdma_cm_event:
+        rdma_cm_id          *id
+        rdma_cm_id          *listen_id
+        rdma_cm_event_type  event
+        int                 status
+        param               param
+
+    cdef struct rdma_addrinfo:
+        int             ai_flags
+        int             ai_family
+        int             ai_qp_type
+        int             ai_port_space
+        int             ai_src_len
+        int             ai_dst_len
+        sockaddr        *ai_src_addr
+        sockaddr        *ai_dst_addr
+        char            *ai_src_canonname
+        char            *ai_dst_canonname
+        size_t          ai_route_len
+        void            *ai_route
+        size_t          ai_connect_len
+        void            *ai_connect
+        rdma_addrinfo   *ai_next
+
+# These non rdmacm structs defined in one of rdma_cma.h's included header files
+    cdef struct sockaddr:
+        unsigned short  sa_family
+        char            sa_data[14]
+
+    cdef struct in_addr:
+        uint32_t s_addr
+
+    cdef struct sockaddr_in:
+        short           sin_family
+        unsigned short  sin_port
+        in_addr         sin_addr
+        char            sin_zero[8]
+
+    int rdma_create_ep(rdma_cm_id **id, rdma_addrinfo *res,
+                       ibv_pd *pd, ibv_qp_init_attr *qp_init_attr)
+    void rdma_destroy_ep(rdma_cm_id *id)
+    int rdma_get_request(rdma_cm_id *listen, rdma_cm_id **id)
+    int rdma_connect(rdma_cm_id *id, rdma_conn_param *conn_param)
+    int rdma_disconnect(rdma_cm_id *id)
+    int rdma_listen(rdma_cm_id *id, int backlog)
+    int rdma_accept(rdma_cm_id *id, rdma_conn_param *conn_param)
+    int rdma_getaddrinfo(char *node, char *service, rdma_addrinfo *hints,
+                         rdma_addrinfo **res)
+    void rdma_freeaddrinfo(rdma_addrinfo *res)
+
+cdef extern from '<rdma/rdma_verbs.h>':
+    int rdma_post_recv(rdma_cm_id *id, void *context, void *addr,
+                       size_t length, ibv_mr *mr)
+    int rdma_post_send(rdma_cm_id *id, void *context, void *addr,
+                       size_t length, ibv_mr *mr, int flags)
+    int rdma_get_send_comp(rdma_cm_id *id, ibv_wc *wc)
+    int rdma_get_recv_comp(rdma_cm_id *id, ibv_wc *wc)
+    ibv_mr *rdma_reg_msgs(rdma_cm_id *id, void *addr, size_t length)
+    int rdma_dereg_mr(ibv_mr *mr)
diff --git a/pyverbs/librdmacm_enums.pxd b/pyverbs/librdmacm_enums.pxd
new file mode 100755
index 000000000000..22a3648fb4a6
--- /dev/null
+++ b/pyverbs/librdmacm_enums.pxd
@@ -0,0 +1,32 @@
+cdef extern from '<rdma/rdma_cma.h>':
+
+    cpdef enum rdma_cm_event_type:
+        RDMA_CM_EVENT_ADDR_RESOLVED
+        RDMA_CM_EVENT_ADDR_ERROR
+        RDMA_CM_EVENT_ROUTE_RESOLVED
+        RDMA_CM_EVENT_ROUTE_ERROR
+        RDMA_CM_EVENT_CONNECT_REQUEST
+        RDMA_CM_EVENT_CONNECT_RESPONSE
+        RDMA_CM_EVENT_CONNECT_ERROR
+        RDMA_CM_EVENT_UNREACHABLE
+        RDMA_CM_EVENT_REJECTED
+        RDMA_CM_EVENT_ESTABLISHED
+        RDMA_CM_EVENT_DISCONNECTED
+        RDMA_CM_EVENT_DEVICE_REMOVAL
+        RDMA_CM_EVENT_MULTICAST_JOIN
+        RDMA_CM_EVENT_MULTICAST_ERROR
+        RDMA_CM_EVENT_ADDR_CHANGE
+        RDMA_CM_EVENT_TIMEWAIT_EXIT
+
+    cpdef enum rdma_port_space:
+        RDMA_PS_IPOIB
+        RDMA_PS_TCP
+        RDMA_PS_UDP
+        RDMA_PS_IB
+
+    # Hint flags which control the operation.
+    cpdef enum:
+        RAI_PASSIVE
+        RAI_NUMERICHOST
+        RAI_NOROUTE
+        RAI_FAMILY
diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx
old mode 100644
new mode 100755
index 46cbb36009ce..d6af58f25980
--- a/pyverbs/pd.pyx
+++ b/pyverbs/pd.pyx
@@ -2,9 +2,10 @@
 # Copyright (c) 2019, Mellanox Technologies. All rights reserved.
 import weakref
 
-from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
 from pyverbs.base import PyverbsRDMAErrno
 from pyverbs.device cimport Context, DM
+from pyverbs.cmid cimport CMID
 from .mr cimport MR, MW, DMMR
 from pyverbs.srq cimport SRQ
 from pyverbs.addr cimport AH
@@ -15,18 +16,27 @@ cdef extern from 'errno.h':
 
 
 cdef class PD(PyverbsCM):
-    def __cinit__(self, Context context not None):
+    def __cinit__(self, object creator not None):
         """
         Initializes a PD object. A reference for the creating Context is kept
         so that Python's GC will destroy the objects in the right order.
         :param context: The Context object creating the PD
         :return: The newly created PD on success
         """
-        self.pd = v.ibv_alloc_pd(<v.ibv_context*>context.context)
-        if self.pd == NULL:
-            raise PyverbsRDMAErrno('Failed to allocate PD', errno)
-        self.ctx = context
-        context.add_ref(self)
+        if issubclass(type(creator), Context):
+            self.pd = v.ibv_alloc_pd((<Context>creator).context)
+            if self.pd == NULL:
+                raise PyverbsRDMAErrno('Failed to allocate PD')
+            self.ctx = creator
+        elif issubclass(type(creator), CMID):
+            cmid = <CMID>creator
+            self.pd = cmid.id.pd
+            self.ctx = cmid.ctx
+            cmid.pd = self
+        else:
+            raise PyverbsUserError('Cannot create PD from {type}'
+                                    .format(type=type(creator)))
+        self.ctx.add_ref(self)
         self.logger.debug('PD: Allocated ibv_pd')
         self.srqs = weakref.WeakSet()
         self.mrs = weakref.WeakSet()
-- 
2.21.0

[PATCH rdma-core 2/5] tests: Fix PD API test

[Noa Osherovich]

From: Maxim Chicherin <maximc@mellanox.com>

In order to support PD class in CMID, modifications was made in PD
constructor, as a result one of test_pd's cases broke.
Fix it to create PD properly.

Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
 tests/test_pd.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
 mode change 100644 => 100755 tests/test_pd.py

diff --git a/tests/test_pd.py b/tests/test_pd.py
old mode 100644
new mode 100755
index 978cf4900146..a8d6eb2fb69f
--- a/tests/test_pd.py
+++ b/tests/test_pd.py
@@ -48,8 +48,7 @@ class PDTest(PyverbsAPITestCase):
         try:
             PD(None)
         except TypeError as te:
-            assert 'expected pyverbs.device.Context' in te.args[0]
-            assert 'got NoneType' in te.args[0]
+            assert 'must not be None' in te.args[0]
         else:
             raise PyverbsRDMAErrno('Created a PD with None context')
 
-- 
2.21.0

[PATCH rdma-core 3/5] tests: New CMResources Class

[Noa Osherovich]

From: Maxim Chicherin <maximc@mellanox.com>

A base aggregation object for RDMACM. Currently only synchronous data
path flow is supported.

Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
 tests/base.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/tests/base.py b/tests/base.py
index e4e92fa2a35f..54bfc42ffe4c 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -11,13 +11,16 @@ from pyverbs.qp import QPCap, QPInitAttrEx, QPInitAttr, QPAttr, QP
 from pyverbs.addr import AHAttr, GlobalRoute
 from pyverbs.xrcd import XRCD, XRCDInitAttr
 from pyverbs.srq import SRQ, SrqInitAttrEx
+from pyverbs.cmid import CMID, AddrInfo
 from pyverbs.device import Context
+import pyverbs.cm_enums as ce
 import pyverbs.device as d
 import pyverbs.enums as e
 from pyverbs.pd import PD
 from pyverbs.cq import CQ
 from pyverbs.mr import MR
 
+
 PATH_MTU = e.IBV_MTU_1024
 MAX_DEST_RD_ATOMIC = 1
 MAX_RD_ATOMIC = 1
@@ -26,6 +29,10 @@ RETRY_CNT = 7
 RNR_RETRY = 7
 TIMEOUT = 14
 
+# for rdmacm
+PORT = '7471'
+ZERO_ADDR = '0.0.0.0'
+
 
 class PyverbsAPITestCase(unittest.TestCase):
     def setUp(self):
@@ -144,6 +151,51 @@ class RDMATestCase(unittest.TestCase):
             self._add_gids_per_port(ctx, dev, port+1)
 
 
+class CMResources:
+    """
+    CMResources class is a base aggregator object which contains basic
+    resources for RDMA CM communication.
+    """
+    def __init__(self, src=ZERO_ADDR, dst=ZERO_ADDR, port=PORT):
+        """
+        :param src: Local address to bind to (for passive side)
+        :param dst: Destination address to connect (for active side)
+        :param port: Port number of the address
+        """
+        self.is_server = True if dst == ZERO_ADDR else False
+        self.qp_init_attr = None
+        self.msg_size = 1024
+        self.num_msgs = 100
+        self.new_id = None
+        self.port = port
+        self.mr = None
+        if self.is_server:
+            self.ai = AddrInfo(src, self.port, ce.RDMA_PS_TCP,
+                               ce.RAI_PASSIVE)
+        else:
+            self.ai = AddrInfo(dst, self.port, ce.RDMA_PS_TCP)
+        self.create_qp_init_attr()
+        self.cmid = CMID(creator=self.ai, qp_init_attr=self.qp_init_attr)
+
+    def create_mr(self, cmid):
+        self.mr = cmid.reg_msgs(self.msg_size)
+
+    def create_qp_init_attr(self):
+        self.qp_init_attr = QPInitAttr(cap=QPCap(max_recv_wr=1))
+
+    def listen(self):
+        self.cmid.listen()
+        self.new_id = self.cmid.get_request()
+        self.new_id.accept()
+
+    def pre_run(self):
+        if self.is_server:
+            self.listen()
+        else:
+            self.cmid.connect()
+        self.create_mr(self.new_id if self.is_server else self.cmid)
+
+
 class BaseResources(object):
     """
     BaseResources class is a base aggregator object which contains basic
-- 
2.21.0

[PATCH rdma-core 4/5] tests: Add RDMACM synchronous traffic test

[Noa Osherovich]

From: Maxim Chicherin <maximc@mellanox.com>

This test creates active and passive sides for RDMACM communication
and validates received messages.

Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
 tests/CMakeLists.txt  |  2 ++
 tests/rdmacm_utils.py | 43 ++++++++++++++++++++++++++++++++
 tests/test_rdmacm.py  | 57 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 102 insertions(+)
 mode change 100644 => 100755 tests/CMakeLists.txt
 create mode 100755 tests/rdmacm_utils.py
 create mode 100755 tests/test_rdmacm.py

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
old mode 100644
new mode 100755
index 960276230860..0d81d1a98fb7
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -3,6 +3,7 @@
 
 rdma_python_test(tests
   __init__.py
+  rdmacm_utils.py
   test_addr.py
   base.py
   test_cq.py
@@ -11,6 +12,7 @@ rdma_python_test(tests
   test_pd.py
   test_qp.py
   test_odp.py
+  test_rdmacm.py
   utils.py
   )
 
diff --git a/tests/rdmacm_utils.py b/tests/rdmacm_utils.py
new file mode 100755
index 000000000000..59b627393e4f
--- /dev/null
+++ b/tests/rdmacm_utils.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.  See COPYING file
+"""
+Provide some useful helper function for pyverbs rdmacm' tests.
+"""
+from tests.base import CMResources
+from tests.utils import validate
+
+
+def active_side(node):
+    client = CMResources(dst=node)
+    client.pre_run()
+    connected_id = client.cmid
+    send_msg = 'c' * client.msg_size
+    for _ in range(client.num_msgs):
+        client.mr.write(send_msg, client.msg_size)
+        connected_id.post_send(client.mr)
+        while connected_id.get_send_comp() is None:
+            pass
+        connected_id.post_recv(client.mr)
+        while connected_id.get_recv_comp() is None:
+            pass
+        msg_received = client.mr.read(client.msg_size, 0)
+        validate(msg_received, False, client.msg_size)
+    connected_id.disconnect()
+
+
+def passive_side(node):
+    server = CMResources(src=node)
+    server.pre_run()
+    connected_id = server.new_id
+    send_msg = 's' * server.msg_size
+    for _ in range(server.num_msgs):
+        connected_id.post_recv(server.mr)
+        while connected_id.get_recv_comp() is None:
+            pass
+        msg_received = server.mr.read(server.msg_size, 0)
+        validate(msg_received, True, server.msg_size)
+        server.mr.write(send_msg, server.msg_size)
+        connected_id.post_send(server.mr)
+        while connected_id.get_send_comp() is None:
+            pass
+    connected_id.disconnect()
diff --git a/tests/test_rdmacm.py b/tests/test_rdmacm.py
new file mode 100755
index 000000000000..e435c635d7b2
--- /dev/null
+++ b/tests/test_rdmacm.py
@@ -0,0 +1,57 @@
+from tests.rdmacm_utils import active_side, passive_side
+from tests.base import RDMATestCase
+from multiprocessing import Process
+import pyverbs.device as d
+import subprocess
+import unittest
+import json
+
+
+class CMTestCase(RDMATestCase):
+    def setUp(self):
+        if self.dev_name is not None:
+            net_name = self.get_net_name(self.dev_name)
+            try:
+                self.ip_addr = self.get_ip_address(net_name)
+            except KeyError:
+                raise unittest.SkipTest('Device {} has no net interface'
+                                        .format(self.dev_name))
+        else:
+            dev_list = d.get_device_list()
+            for dev in dev_list:
+                net_name = self.get_net_name(dev.name.decode())
+                try:
+                    self.ip_addr = self.get_ip_address(net_name)
+                except IndexError:
+                    continue
+                else:
+                    self.dev_name = dev.name.decode()
+                    break
+            if self.dev_name is None:
+                raise unittest.SkipTest('No devices with net interface')
+        super().setUp()
+
+    @staticmethod
+    def get_net_name(dev):
+        process = subprocess.Popen(['ls', '/sys/class/infiniband/{}/device/net/'
+                                   .format(dev)], stdout=subprocess.PIPE)
+        out, err = process.communicate()
+        return out.decode().split('\n')[0]
+
+    @staticmethod
+    def get_ip_address(ifname):
+        process = subprocess.Popen(['ip', '-j', 'addr', 'show', ifname],
+                                   stdout=subprocess.PIPE)
+        out, err = process.communicate()
+        loaded_json = json.loads(out.decode())
+        interface = loaded_json[0]['addr_info'][0]['local']
+        if 'fe80::' in interface:
+            interface = interface + '%' + ifname
+        return interface
+
+    def test_rdmacm_sync_traffic(self):
+        ps = Process(target=passive_side, args=[self.ip_addr])
+        ps.start()
+        ps.join(1)
+        active_side(self.ip_addr)
+        ps.join()
-- 
2.21.0

[PATCH rdma-core 5/5] Documentation: Document creation of CMID

[Noa Osherovich]

From: Maxim Chicherin <maximc@mellanox.com>

Add code snippet to demonstrate creation of CMID and establish
connection (synchronous control path).

Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
---
 Documentation/pyverbs.md | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md
index 29ab9592c53c..6ff6fc5bbc37 100755
--- a/Documentation/pyverbs.md
+++ b/Documentation/pyverbs.md
@@ -390,3 +390,33 @@ srq_attr.comp_mask = e.IBV_SRQ_INIT_ATTR_TYPE | e.IBV_SRQ_INIT_ATTR_PD | \
                      e.IBV_SRQ_INIT_ATTR_CQ | e.IBV_SRQ_INIT_ATTR_XRCD
 srq = SRQ(ctx, srq_attr)
 ```
+
+##### CMID
+The following code snippet will demonstrate creation of a CMID object, which
+represents rdma_cm_id C struct, and establish connection between two peers.
+Currently only synchronous control path is supported (rdma_create_ep).
+For more complex examples, please see tests/test_rdmacm.
+```python
+from pyverbs.qp import QPInitAttr, QPCap
+from pyverbs.cmid import CMID, AddrInfo
+import pyverbs.cm_enums as ce
+
+cap = QPCap(max_recv_wr=1)
+qp_init_attr = QPInitAttr(cap=cap)
+server = '11.137.14.124'
+port = '7471'
+
+# Active side
+
+cai = AddrInfo(server, port, ce.RDMA_PS_TCP)
+cid = CMID(creator=cai, qp_init_attr=qp_init_attr)
+cid.connect()  # send connection request to server
+
+# Passive side
+
+sai = AddrInfo(server, port, ce.RDMA_PS_TCP, ce.RAI_PASSIVE)
+sid = CMID(creator=sai, qp_init_attr=qp_init_attr)
+sid.listen()  # listen for incoming connection requests
+new_id = sid.get_request()  # check if there are any connection requests
+new_id.accept()  # new_id is connected to remote peer and ready to communicate
+```
-- 
2.21.0

Re: [PATCH rdma-core 1/5] pyverbs: New CMID class

[Benjamin Drung]

Am Montag, den 04.11.2019, 10:37 +0000 schrieb Noa Osherovich:
> diff --git a/pyverbs/cm_enums.pyx b/pyverbs/cm_enums.pyx
> new file mode 120000
> index 000000000000..bdab2b585a1d
> --- /dev/null
> +++ b/pyverbs/cm_enums.pyx
> @@ -0,0 +1 @@
> +librdmacm_enums.pxd
> \ No newline at end of file
> diff --git a/pyverbs/cmid.pxd b/pyverbs/cmid.pxd
> new file mode 100755
> index 000000000000..56bc755daf42
> --- /dev/null
> +++ b/pyverbs/cmid.pxd
> @@ -0,0 +1,25 @@
> +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> +# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
> See COPYING file
> +
> +#cython: language_level=3
> +
> +from pyverbs.base cimport PyverbsObject, PyverbsCM
> +from libc.string cimport memcpy, memset
> +from libc.stdlib cimport free, malloc
> +cimport pyverbs.librdmacm as cm
> +
> +
> +cdef class CMID(PyverbsCM):
> +    cdef cm.rdma_cm_id *id
> +    cdef object ctx
> +    cdef object pd
> +    cpdef close(self)
> +
> +
> +cdef class AddrInfo(PyverbsObject):
> +    cdef cm.rdma_addrinfo *addr_info
> +    cpdef close(self)
> +
> +
> +cdef class ConnParam(PyverbsObject):
> +    cdef cm.rdma_conn_param conn_param
> \ No newline at end of file

Please add newlines to the end of these files.

-- 
Benjamin Drung

Debian & Ubuntu Developer
Platform Engineering Compute (Enterprise Cloud)

1&1 IONOS SE | Greifswalder Str. 207 | 10405 Berlin | Germany
E-mail: benjamin.drung@cloud.ionos.com | Web: www.ionos.de

Hauptsitz Montabaur, Amtsgericht Montabaur, HRB 24498
Vorstand: Dr. Christian Böing, Hüseyin Dogan, Hans-Henning Kettler,
Matthias Steinberg, Achim Weiß
Aufsichtsratsvorsitzender: Markus Kadelke
Member of United Internet

Re: [PATCH rdma-core 1/5] pyverbs: New CMID class

[Leon Romanovsky]

On Mon, Nov 04, 2019 at 12:09:59PM +0100, Benjamin Drung wrote:
> Am Montag, den 04.11.2019, 10:37 +0000 schrieb Noa Osherovich:
> > diff --git a/pyverbs/cm_enums.pyx b/pyverbs/cm_enums.pyx
> > new file mode 120000
> > index 000000000000..bdab2b585a1d
> > --- /dev/null
> > +++ b/pyverbs/cm_enums.pyx
> > @@ -0,0 +1 @@
> > +librdmacm_enums.pxd
> > \ No newline at end of file
> > diff --git a/pyverbs/cmid.pxd b/pyverbs/cmid.pxd
> > new file mode 100755
> > index 000000000000..56bc755daf42
> > --- /dev/null
> > +++ b/pyverbs/cmid.pxd
> > @@ -0,0 +1,25 @@
> > +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> > +# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
> > See COPYING file
> > +
> > +#cython: language_level=3
> > +
> > +from pyverbs.base cimport PyverbsObject, PyverbsCM
> > +from libc.string cimport memcpy, memset
> > +from libc.stdlib cimport free, malloc
> > +cimport pyverbs.librdmacm as cm
> > +
> > +
> > +cdef class CMID(PyverbsCM):
> > +    cdef cm.rdma_cm_id *id
> > +    cdef object ctx
> > +    cdef object pd
> > +    cpdef close(self)
> > +
> > +
> > +cdef class AddrInfo(PyverbsObject):
> > +    cdef cm.rdma_addrinfo *addr_info
> > +    cpdef close(self)
> > +
> > +
> > +cdef class ConnParam(PyverbsObject):
> > +    cdef cm.rdma_conn_param conn_param
> > \ No newline at end of file
>
> Please add newlines to the end of these files.

PR was updated.

Thanks

>
> --
> Benjamin Drung
>
> Debian & Ubuntu Developer
> Platform Engineering Compute (Enterprise Cloud)
>
> 1&1 IONOS SE | Greifswalder Str. 207 | 10405 Berlin | Germany
> E-mail: benjamin.drung@cloud.ionos.com | Web: www.ionos.de
>
> Hauptsitz Montabaur, Amtsgericht Montabaur, HRB 24498
> Vorstand: Dr. Christian Böing, Hüseyin Dogan, Hans-Henning Kettler,
> Matthias Steinberg, Achim Weiß
> Aufsichtsratsvorsitzender: Markus Kadelke
> Member of United Internet
>