[PATCH rdma-core 5/7] pyverbs/mlx5: Add support for mlx5 CQ

[Noa Osherovich <noaos@mellanox.com>]

Add the needed infrastructure to allow users to create a CQ via the
mlx5dv interface.
Creation process requires CQInitAttrEx, similarly to the creation of
an extended CQ, but requires an Mlx5DVCQInitAttr object as well
(provided as kwargs, i.e. dv_init_attr=<your object>).

Signed-off-by: Noa Osherovich <noaos@mellanox.com>
---
 pyverbs/providers/mlx5/libmlx5.pxd      |   9 +++
 pyverbs/providers/mlx5/mlx5dv.pxd       |   7 ++
 pyverbs/providers/mlx5/mlx5dv.pyx       | 103 ++++++++++++++++++++++++
 pyverbs/providers/mlx5/mlx5dv_enums.pxd |   9 +++
 4 files changed, 128 insertions(+)

diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index 2c5be241a3ec..f471f2453e38 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -52,6 +52,12 @@ cdef extern from 'infiniband/mlx5dv.h':
         mlx5dv_dc_init_attr dc_init_attr
         unsigned long       send_ops_flags
 
+    cdef struct mlx5dv_cq_init_attr:
+        unsigned long   comp_mask
+        unsigned char   cqe_comp_res_format
+        unsigned int    flags
+        unsigned short  cqe_size
+
     bool mlx5dv_is_supported(v.ibv_device *device)
     v.ibv_context* mlx5dv_open_device(v.ibv_device *device,
                                       mlx5dv_context_attr *attr)
@@ -59,3 +65,6 @@ cdef extern from 'infiniband/mlx5dv.h':
     v.ibv_qp *mlx5dv_create_qp(v.ibv_context *context,
                                v.ibv_qp_init_attr_ex *qp_attr,
                                mlx5dv_qp_init_attr *mlx5_qp_attr)
+    v.ibv_cq_ex *mlx5dv_create_cq(v.ibv_context *context,
+                                  v.ibv_cq_init_attr_ex *cq_attr,
+                                  mlx5dv_cq_init_attr *mlx5_cq_attr)
diff --git a/pyverbs/providers/mlx5/mlx5dv.pxd b/pyverbs/providers/mlx5/mlx5dv.pxd
index 918065012d30..cc9c89d82fa6 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv.pxd
@@ -6,6 +6,7 @@
 cimport pyverbs.providers.mlx5.libmlx5 as dv
 from pyverbs.base cimport PyverbsObject
 from pyverbs.device cimport Context
+from pyverbs.cq cimport CQEX
 from pyverbs.qp cimport QP
 
 
@@ -26,3 +27,9 @@ cdef class Mlx5DVQPInitAttr(PyverbsObject):
 
 cdef class Mlx5QP(QP):
     cdef object dc_type
+
+cdef class Mlx5DVCQInitAttr(PyverbsObject):
+    cdef dv.mlx5dv_cq_init_attr attr
+
+cdef class Mlx5CQ(CQEX):
+    pass
diff --git a/pyverbs/providers/mlx5/mlx5dv.pyx b/pyverbs/providers/mlx5/mlx5dv.pyx
index b4a971a5b935..e4500567ba23 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pyx
+++ b/pyverbs/providers/mlx5/mlx5dv.pyx
@@ -7,6 +7,8 @@ cimport pyverbs.providers.mlx5.libmlx5 as dv
 from pyverbs.base import PyverbsRDMAErrno
 cimport pyverbs.libibverbs_enums as e
 from pyverbs.qp cimport QPInitAttrEx
+from pyverbs.cq cimport CqInitAttrEx
+cimport pyverbs.libibverbs as v
 from pyverbs.pd cimport PD
 
 cdef class Mlx5DVContextAttr(PyverbsObject):
@@ -331,6 +333,107 @@ cdef class Mlx5QP(QP):
         return masks[self.dc_type][dst] | e.IBV_QP_STATE
 
 
+cdef class Mlx5DVCQInitAttr(PyverbsObject):
+    """
+    Represents mlx5dv_cq_init_attr struct, initial attributes used for mlx5 CQ
+    creation.
+    """
+    def __cinit__(self, comp_mask=0, cqe_comp_res_format=0, flags=0, cqe_size=0):
+        """
+        Initializes an Mlx5CQInitAttr object with zeroes as default values.
+        :param comp_mask: Marks which of the following fields should be
+                          considered. Use mlx5dv_cq_init_attr_mask enum.
+        :param cqe_comp_res_format: The various CQE response formats of the
+                                    responder side. Use
+                                    mlx5dv_cqe_comp_res_format enum.
+        :param flags: A bitwise OR of the various values described in
+                      mlx5dv_cq_init_attr_flags.
+        :param cqe_size: Configure the CQE size to be 64 or 128 bytes, other
+                         values will cause the CQ creation process to fail.
+                         Valid when MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE is set.
+        :return: None
+        """
+        self.attr.comp_mask = comp_mask
+        self.attr.cqe_comp_res_format = cqe_comp_res_format
+        self.attr.flags = flags
+        self.attr.cqe_size = cqe_size
+
+    @property
+    def comp_mask(self):
+        return self.attr.comp_mask
+    @comp_mask.setter
+    def comp_mask(self, val):
+        self.attr.comp_mask = val
+
+    @property
+    def cqe_comp_res_format(self):
+        return self.attr.cqe_comp_res_format
+    @cqe_comp_res_format.setter
+    def cqe_comp_res_format(self, val):
+        self.attr.cqe_comp_res_format = val
+
+    @property
+    def flags(self):
+        return self.attr.flags
+    @flags.setter
+    def flags(self, val):
+        self.attr.flags = val
+
+    @property
+    def cqe_size(self):
+        return self.attr.cqe_size
+    @cqe_size.setter
+    def cqe_size(self, val):
+        self.attr.cqe_size = val
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        flags = {dve.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD:
+                     "MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD}"}
+        mask = {dve.MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE:
+                    "MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE",
+                dve.MLX5DV_CQ_INIT_ATTR_MASK_FLAGS:
+                    "MLX5DV_CQ_INIT_ATTR_MASK_FLAGS",
+                dve.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE:
+                    "MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE"}
+        fmt = {dve.MLX5DV_CQE_RES_FORMAT_HASH: "MLX5DV_CQE_RES_FORMAT_HASH",
+               dve.MLX5DV_CQE_RES_FORMAT_CSUM: "MLX5DV_CQE_RES_FORMAT_CSUM",
+               dve.MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX:
+                   "MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX"}
+
+        return 'Mlx5DVCQInitAttr:\n' +\
+               print_format.format('comp_mask', bitmask_to_str(self.comp_mask,
+                                                               mask)) +\
+               print_format.format('CQE compression format',
+                                   bitmask_to_str(self.cqe_comp_res_format,
+                                                  fmt)) +\
+               print_format.format('flags', bitmask_to_str(self.flags,
+                                                           flags)) + \
+               print_format.format('CQE size', self.cqe_size)
+
+
+cdef class Mlx5CQ(CQEX):
+    def __cinit__(self, Mlx5Context context, CqInitAttrEx init_attr,
+                  Mlx5DVCQInitAttr dv_init_attr):
+        self.cq = \
+            dv.mlx5dv_create_cq(context.context, &init_attr.attr,
+                                &dv_init_attr.attr if dv_init_attr is not None
+                                else NULL)
+        if self.cq == NULL:
+            raise PyverbsRDMAErrno('Failed to create MLX5 CQ.\nCQInitAttrEx:\n'
+                                   '{}\nMLX5DVCQInitAttr:\n{}'.
+                                   format(init_attr, dv_init_attr))
+        self.ibv_cq = v.ibv_cq_ex_to_cq(self.cq)
+        self.context = context
+        context.add_ref(self)
+
+    def __str__(self):
+        print_format = '{:<22}: {:<20}\n'
+        return 'Mlx5 CQ:\n' +\
+               print_format.format('Handle', self.cq.handle) +\
+               print_format.format('CQEs', self.cq.cqe)
+
+
 def qpts_to_str(qp_types):
     numeric_types = qp_types
     qpts_str = ''
diff --git a/pyverbs/providers/mlx5/mlx5dv_enums.pxd b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
index 3f7f591e23a5..3859dff81ad1 100644
--- a/pyverbs/providers/mlx5/mlx5dv_enums.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
@@ -66,3 +66,12 @@ cdef extern from 'infiniband/mlx5dv.h':
     cpdef enum mlx5dv_qp_create_send_ops_flags:
         MLX5DV_QP_EX_WITH_MR_INTERLEAVED    = 1 << 0
         MLX5DV_QP_EX_WITH_MR_LIST           = 1 << 1
+
+    cpdef enum mlx5dv_cq_init_attr_mask:
+        MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0
+        MLX5DV_CQ_INIT_ATTR_MASK_FLAGS          = 1 << 1
+        MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE       = 1 << 2
+
+    cpdef enum mlx5dv_cq_init_attr_flags:
+        MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD   = 1 << 0
+        MLX5DV_CQ_INIT_ATTR_FLAGS_RESERVED  = 1 << 1
-- 
2.21.0