[PATCH rdma-core 22/27] pyverbs/mlx5: Add support to extract mlx5dv objects

[Yishai Hadas <yishaih@nvidia.com>]

From: Edward Srouji <edwards@nvidia.com>

Support extraction of mlx5dv objects from ibv objects.
This allows the users to access the object numbers and much more
attributes needed for some usage such as DevX.
Currently there's a support for the following objects: PD, CQ, QP and
SRQ.

Reviewed-by: Ido Kalir <idok@nvidia.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>
---
 pyverbs/pd.pyx                            |   4 +
 pyverbs/providers/mlx5/CMakeLists.txt     |   1 +
 pyverbs/providers/mlx5/libmlx5.pxd        |  52 ++++++++
 pyverbs/providers/mlx5/mlx5dv_enums.pxd   |  17 +++
 pyverbs/providers/mlx5/mlx5dv_objects.pxd |  28 ++++
 pyverbs/providers/mlx5/mlx5dv_objects.pyx | 214 ++++++++++++++++++++++++++++++
 6 files changed, 316 insertions(+)
 create mode 100644 pyverbs/providers/mlx5/mlx5dv_objects.pxd
 create mode 100644 pyverbs/providers/mlx5/mlx5dv_objects.pyx

diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx
index 2c0c424..e8d3e1d 100644
--- a/pyverbs/pd.pyx
+++ b/pyverbs/pd.pyx
@@ -139,6 +139,10 @@ cdef class PD(PyverbsCM):
     def handle(self):
         return self.pd.handle
 
+    @property
+    def pd(self):
+        return <object>self.pd
+
 
 cdef void *pd_alloc(v.ibv_pd *pd, void *pd_context, size_t size,
                   size_t alignment, v.uint64_t resource_type):
diff --git a/pyverbs/providers/mlx5/CMakeLists.txt b/pyverbs/providers/mlx5/CMakeLists.txt
index cdb6be2..4763c61 100644
--- a/pyverbs/providers/mlx5/CMakeLists.txt
+++ b/pyverbs/providers/mlx5/CMakeLists.txt
@@ -11,5 +11,6 @@ rdma_cython_module(pyverbs/providers/mlx5 mlx5
   mlx5_enums.pyx
   mlx5dv_flow.pyx
   mlx5dv_mkey.pyx
+  mlx5dv_objects.pyx
   mlx5dv_sched.pyx
 )
diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index 34691a9..de4008d 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -4,6 +4,7 @@
 include 'mlx5dv_enums.pxd'
 
 from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, uintptr_t
+from posix.types cimport off_t
 from libcpp cimport bool
 
 cimport pyverbs.libibverbs as v
@@ -222,6 +223,56 @@ cdef extern from 'infiniband/mlx5dv.h':
         uint64_t    pgsz_bitmap
         uint64_t    comp_mask
 
+    cdef struct mlx5dv_pd:
+        uint32_t    pdn
+        uint64_t    comp_mask
+
+    cdef struct mlx5dv_cq:
+        void        *buf
+        uint32_t    *dbrec
+        uint32_t    cqe_cnt
+        uint32_t    cqe_size
+        void        *cq_uar
+        uint32_t    cqn
+        uint64_t    comp_mask
+
+    cdef struct mlx5dv_qp:
+        uint64_t    comp_mask
+        off_t       uar_mmap_offset
+        uint32_t    tirn
+        uint32_t    tisn
+        uint32_t    rqn
+        uint32_t    sqn
+
+    cdef struct mlx5dv_srq:
+        uint32_t    stride
+        uint32_t    head
+        uint32_t    tail
+        uint64_t    comp_mask
+        uint32_t    srqn
+
+    cdef struct pd:
+        v.ibv_pd    *in_ "in"
+        mlx5dv_pd   *out
+
+    cdef struct cq:
+        v.ibv_cq    *in_ "in"
+        mlx5dv_cq   *out
+
+    cdef struct qp:
+        v.ibv_qp    *in_ "in"
+        mlx5dv_qp   *out
+
+    cdef struct srq:
+        v.ibv_srq   *in_ "in"
+        mlx5dv_srq  *out
+
+    cdef struct mlx5dv_obj:
+        pd  pd
+        cq  cq
+        qp  qp
+        srq srq
+
 
     void mlx5dv_set_ctrl_seg(mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode,
                              uint8_t opmod, uint32_t qp_num, uint8_t fm_ce_se,
@@ -326,6 +377,7 @@ cdef extern from 'infiniband/mlx5dv.h':
     int mlx5dv_devx_obj_modify(mlx5dv_devx_obj *obj, const void *in_,
                                size_t inlen, void *out, size_t outlen)
     int mlx5dv_devx_obj_destroy(mlx5dv_devx_obj *obj)
+    int mlx5dv_init_obj(mlx5dv_obj *obj, uint64_t obj_type)
 
     # Mkey setters
     void mlx5dv_wr_mkey_configure(mlx5dv_qp_ex *mqp, mlx5dv_mkey *mkey,
diff --git a/pyverbs/providers/mlx5/mlx5dv_enums.pxd b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
index d0a29bc..9f8d1a1 100644
--- a/pyverbs/providers/mlx5/mlx5dv_enums.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
@@ -176,6 +176,23 @@ cdef extern from 'infiniband/mlx5dv.h':
         MLX5DV_DR_DOMAIN_TYPE_NIC_TX
         MLX5DV_DR_DOMAIN_TYPE_FDB
 
+    cpdef enum mlx5dv_qp_comp_mask:
+        MLX5DV_QP_MASK_UAR_MMAP_OFFSET
+        MLX5DV_QP_MASK_RAW_QP_HANDLES
+        MLX5DV_QP_MASK_RAW_QP_TIR_ADDR
+
+    cpdef enum mlx5dv_srq_comp_mask:
+        MLX5DV_SRQ_MASK_SRQN
+
+    cpdef enum mlx5dv_obj_type:
+        MLX5DV_OBJ_QP
+        MLX5DV_OBJ_CQ
+        MLX5DV_OBJ_SRQ
+        MLX5DV_OBJ_RWQ
+        MLX5DV_OBJ_DM
+        MLX5DV_OBJ_AH
+        MLX5DV_OBJ_PD
+
     cpdef unsigned long long MLX5DV_RES_TYPE_QP
     cpdef unsigned long long MLX5DV_RES_TYPE_RWQ
     cpdef unsigned long long MLX5DV_RES_TYPE_DBR
diff --git a/pyverbs/providers/mlx5/mlx5dv_objects.pxd b/pyverbs/providers/mlx5/mlx5dv_objects.pxd
new file mode 100644
index 0000000..e8eab84
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5dv_objects.pxd
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+cimport pyverbs.providers.mlx5.libmlx5 as dv
+from pyverbs.base cimport PyverbsObject
+
+
+cdef class Mlx5DvPD(PyverbsObject):
+    cdef dv.mlx5dv_pd dv_pd
+
+cdef class Mlx5DvCQ(PyverbsObject):
+    cdef dv.mlx5dv_cq dv_cq
+
+cdef class Mlx5DvQP(PyverbsObject):
+    cdef dv.mlx5dv_qp dv_qp
+
+cdef class Mlx5DvSRQ(PyverbsObject):
+    cdef dv.mlx5dv_srq dv_srq
+
+cdef class Mlx5DvObj(PyverbsObject):
+    cdef dv.mlx5dv_obj obj
+    cdef Mlx5DvCQ dv_cq
+    cdef Mlx5DvQP dv_qp
+    cdef Mlx5DvPD dv_pd
+    cdef Mlx5DvSRQ dv_srq
+
diff --git a/pyverbs/providers/mlx5/mlx5dv_objects.pyx b/pyverbs/providers/mlx5/mlx5dv_objects.pyx
new file mode 100644
index 0000000..ec6eeb6
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5dv_objects.pyx
@@ -0,0 +1,214 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+"""
+This module wraps mlx5dv_<obj> C structs, such as mlx5dv_cq, mlx5dv_qp etc.
+It exposes to the users the mlx5 driver-specific attributes for ibv objects by
+extracting them via mlx5dv_init_obj() API by using Mlx5DvObj class, which holds
+all the (currently) supported Mlx5Dv<Obj> objects.
+Note: This is not be confused with Mlx5<Obj> which holds the ibv_<obj>_ex that
+      was created using mlx5dv_create_<obj>().
+"""
+
+from libc.stdint cimport uintptr_t, uint32_t
+
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsRDMAError
+cimport pyverbs.providers.mlx5.mlx5dv_enums as dve
+cimport pyverbs.libibverbs as v
+
+
+cdef class Mlx5DvPD(PyverbsObject):
+
+    @property
+    def pdn(self):
+        """ The protection domain object number """
+        return self.dv_pd.pdn
+
+    @property
+    def comp_mask(self):
+        return self.dv_pd.comp_mask
+
+
+cdef class Mlx5DvCQ(PyverbsObject):
+
+    @property
+    def cqe_size(self):
+        return self.dv_cq.cqe_size
+
+    @property
+    def comp_mask(self):
+        return self.dv_cq.comp_mask
+
+    @property
+    def cqn(self):
+        return self.dv_cq.cqn
+
+    @property
+    def buf(self):
+        return <uintptr_t><void*>self.dv_cq.buf
+
+    @property
+    def cq_uar(self):
+        return <uintptr_t><void*>self.dv_cq.cq_uar
+
+    @property
+    def dbrec(self):
+        return <uintptr_t><uint32_t*>self.dv_cq.dbrec
+
+    @property
+    def cqe_cnt(self):
+        return self.dv_cq.cqe_cnt
+
+
+cdef class Mlx5DvQP(PyverbsObject):
+
+    @property
+    def rqn(self):
+        """ The receive queue number of the QP"""
+        return self.dv_qp.rqn
+
+    @property
+    def sqn(self):
+        """ The send queue number of the QP"""
+        return self.dv_qp.sqn
+
+    @property
+    def tirn(self):
+        """
+        The number of the transport interface receive object that attached
+        to the RQ of the QP
+        """
+        return self.dv_qp.tirn
+
+    @property
+    def tisn(self):
+        """
+        The number of the transport interface send object that attached
+        to the SQ of the QP
+        """
+        return self.dv_qp.tisn
+
+    @property
+    def comp_mask(self):
+        return self.dv_qp.comp_mask
+    @comp_mask.setter
+    def comp_mask(self, val):
+        self.dv_qp.comp_mask = val
+
+    @property
+    def uar_mmap_offset(self):
+        return self.uar_mmap_offset
+
+
+cdef class Mlx5DvSRQ(PyverbsObject):
+
+    @property
+    def stride(self):
+        return self.dv_srq.stride
+
+    @property
+    def head(self):
+        return self.dv_srq.stride
+
+    @property
+    def tail(self):
+        return self.dv_srq.stride
+
+    @property
+    def comp_mask(self):
+        return self.dv_srq.comp_mask
+    @comp_mask.setter
+    def comp_mask(self, val):
+        self.dv_srq.comp_mask = val
+
+    @property
+    def srqn(self):
+        """ The shared receive queue object number """
+        return self.dv_srq.srqn
+
+
+cdef class Mlx5DvObj(PyverbsObject):
+    """
+    Mlx5DvObj represents mlx5dv_obj C struct.
+    """
+    def __init__(self, obj_type=None, **kwargs):
+        """
+        Retrieves DV objects from ibv object to be able to extract attributes
+        (such as cqe_size of a CQ).
+        Currently supports CQ, QP, PD and SRQ objects.
+        The initialized objects can be accessed using self.dvobj (e.g. self.dvcq).
+        :param obj_type: Bitmask which defines what objects was provided.
+                         Currently it supports: MLX5DV_OBJ_CQ, MLX5DV_OBJ_QP,
+                         MLX5DV_OBJ_SRQ and MLX5DV_OBJ_PD.
+        :param kwargs: List of objects (cq, qp, pd, srq) from which to extract
+                       data and their comp_masks if applicable. If comp_mask is
+                       not provided by user, mask all by default.
+        """
+        self.dv_pd = self.dv_cq = self.dv_qp = self.dv_srq = None
+        if obj_type is None:
+            return
+        self.init_obj(obj_type, **kwargs)
+
+    def init_obj(self, obj_type, **kwargs):
+        """
+        Initialize DV objects.
+        The objects are re-initialized if they're already extracted.
+        """
+        supported_obj_types = dve.MLX5DV_OBJ_CQ | dve.MLX5DV_OBJ_QP | \
+                         dve.MLX5DV_OBJ_PD | dve.MLX5DV_OBJ_SRQ
+        if obj_type & supported_obj_types is False:
+            raise PyverbsUserError('Invalid obj_type was provided')
+
+        cq = kwargs.get('cq') if obj_type | dve.MLX5DV_OBJ_CQ else None
+        qp = kwargs.get('qp') if obj_type | dve.MLX5DV_OBJ_QP else None
+        pd = kwargs.get('pd') if obj_type | dve.MLX5DV_OBJ_PD else None
+        srq = kwargs.get('srq') if obj_type | dve.MLX5DV_OBJ_SRQ else None
+        if cq is qp is pd is srq is None:
+            raise PyverbsUserError("No supported object was provided.")
+
+        if cq:
+            dv_cq = Mlx5DvCQ()
+            self.obj.cq.in_ = <v.ibv_cq*>cq.cq
+            self.obj.cq.out = &(dv_cq.dv_cq)
+            self.dv_cq = dv_cq
+        if qp:
+            dv_qp = Mlx5DvQP()
+            comp_mask = kwargs.get('qp_comp_mask')
+            dv_qp.comp_mask = comp_mask if comp_mask else \
+                dv.MLX5DV_QP_MASK_UAR_MMAP_OFFSET | \
+                dv.MLX5DV_QP_MASK_RAW_QP_HANDLES | \
+                dv.MLX5DV_QP_MASK_RAW_QP_TIR_ADDR
+            self.obj.qp.in_ = <v.ibv_qp*>qp.qp
+            self.obj.qp.out = &(dv_qp.dv_qp)
+            self.dv_qp = dv_qp
+        if pd:
+            dv_pd = Mlx5DvPD()
+            self.obj.pd.in_ = <v.ibv_pd*>pd.pd
+            self.obj.pd.out = &(dv_pd.dv_pd)
+            self.dv_pd = dv_pd
+        if srq:
+            dv_srq = Mlx5DvSRQ()
+            comp_mask = kwargs.get('srq_comp_mask')
+            dv_srq.comp_mask = comp_mask if comp_mask else dv.MLX5DV_SRQ_MASK_SRQN
+            self.obj.srq.in_ = <v.ibv_srq*>srq.srq
+            self.obj.srq.out = &(dv_srq.dv_srq)
+            self.dv_srq = dv_srq
+
+        rc = dv.mlx5dv_init_obj(&self.obj, obj_type)
+        if rc != 0:
+            raise PyverbsRDMAError("Failed to initialize Mlx5DvObj", rc)
+
+    @property
+    def dvcq(self):
+        return self.dv_cq
+
+    @property
+    def dvqp(self):
+        return self.dv_qp
+
+    @property
+    def dvpd(self):
+        return self.dv_pd
+
+    @property
+    def dvsrq(self):
+        return self.dv_srq
-- 
1.8.3.1