[PATCH rdma-core 26/27] pyverbs/mlx5: Support mlx5 devices over VFIO

[Yishai Hadas <yishaih@nvidia.com>]

From: Edward Srouji <edwards@nvidia.com>

Support opening an mlx5 device and creating a context over VFIO by
adding  Mlx5VfioContext class based on Mlx5Context to allow using DevX
APIs.

Reviewed-by: Ido Kalir <idok@nvidia.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>
---
 pyverbs/providers/mlx5/CMakeLists.txt   |   3 +-
 pyverbs/providers/mlx5/libmlx5.pxd      |   8 +++
 pyverbs/providers/mlx5/mlx5_vfio.pxd    |  15 +++++
 pyverbs/providers/mlx5/mlx5_vfio.pyx    | 116 ++++++++++++++++++++++++++++++++
 pyverbs/providers/mlx5/mlx5dv.pxd       |   3 +
 pyverbs/providers/mlx5/mlx5dv_enums.pxd |   3 +
 6 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 pyverbs/providers/mlx5/mlx5_vfio.pxd
 create mode 100644 pyverbs/providers/mlx5/mlx5_vfio.pyx

diff --git a/pyverbs/providers/mlx5/CMakeLists.txt b/pyverbs/providers/mlx5/CMakeLists.txt
index 4763c61..c0b5869 100644
--- a/pyverbs/providers/mlx5/CMakeLists.txt
+++ b/pyverbs/providers/mlx5/CMakeLists.txt
@@ -7,8 +7,9 @@ rdma_cython_module(pyverbs/providers/mlx5 mlx5
   dr_matcher.pyx
   dr_rule.pyx
   dr_table.pyx
-  mlx5dv.pyx
   mlx5_enums.pyx
+  mlx5_vfio.pyx
+  mlx5dv.pyx
   mlx5dv_flow.pyx
   mlx5dv_mkey.pyx
   mlx5dv_objects.pyx
diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index af034ad..e0904f5 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -223,6 +223,11 @@ cdef extern from 'infiniband/mlx5dv.h':
         uint64_t    pgsz_bitmap
         uint64_t    comp_mask
 
+    cdef struct mlx5dv_vfio_context_attr:
+        const char  *pci_name
+        uint32_t    flags
+        uint64_t    comp_mask
+
     cdef struct mlx5dv_pd:
         uint32_t    pdn
         uint64_t    comp_mask
@@ -372,6 +377,9 @@ cdef extern from 'infiniband/mlx5dv.h':
                              uint64_t device_timestamp)
     int mlx5dv_get_clock_info(v.ibv_context *ctx_in, mlx5dv_clock_info *clock_info)
     int mlx5dv_map_ah_to_qp(v.ibv_ah *ah, uint32_t qp_num)
+    v.ibv_device **mlx5dv_get_vfio_device_list(mlx5dv_vfio_context_attr *attr)
+    int mlx5dv_vfio_get_events_fd(v.ibv_context *ibctx)
+    int mlx5dv_vfio_process_events(v.ibv_context *context)
 
     # DevX APIs
     mlx5dv_devx_uar *mlx5dv_devx_alloc_uar(v.ibv_context *context, uint32_t flags)
diff --git a/pyverbs/providers/mlx5/mlx5_vfio.pxd b/pyverbs/providers/mlx5/mlx5_vfio.pxd
new file mode 100644
index 0000000..0e9facd
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5_vfio.pxd
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from pyverbs.providers.mlx5.mlx5dv cimport Mlx5Context
+cimport pyverbs.providers.mlx5.libmlx5 as dv
+from pyverbs.base cimport PyverbsObject
+
+
+cdef class Mlx5VfioContext(Mlx5Context):
+    pass
+
+cdef class Mlx5VfioAttr(PyverbsObject):
+    cdef dv.mlx5dv_vfio_context_attr attr
diff --git a/pyverbs/providers/mlx5/mlx5_vfio.pyx b/pyverbs/providers/mlx5/mlx5_vfio.pyx
new file mode 100644
index 0000000..2978b61
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5_vfio.pyx
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from libc.string cimport strcpy
+import weakref
+
+from pyverbs.pyverbs_error import PyverbsRDMAError
+cimport pyverbs.providers.mlx5.libmlx5 as dv
+from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.base cimport close_weakrefs
+from pyverbs.device cimport Context
+cimport pyverbs.libibverbs as v
+
+
+cdef class Mlx5VfioAttr(PyverbsObject):
+    """
+    Mlx5VfioAttr class, represents mlx5dv_vfio_context_attr C struct.
+    """
+    def __init__(self, pci_name, flags=0, comp_mask=0):
+        self.pci_name = pci_name
+        self.attr.flags = flags
+        self.attr.comp_mask = comp_mask
+
+    def __dealloc__(self):
+        if self.attr.pci_name != NULL:
+            PyMem_Free(<void*>self.attr.pci_name)
+            self.attr.pci_name = NULL
+
+    @property
+    def flags(self):
+        return self.attr.flags
+    @flags.setter
+    def flags(self, val):
+        self.attr.flags = val
+
+    @property
+    def comp_mask(self):
+        return self.attr.comp_mask
+    @comp_mask.setter
+    def comp_mask(self, val):
+        self.attr.comp_mask = val
+
+    @property
+    def pci_name(self):
+        return self.attr.pci_name[:]
+    @pci_name.setter
+    def pci_name(self, val):
+        if self.attr.pci_name != NULL:
+            PyMem_Free(<void*>self.attr.pci_name)
+        pci_name_bytes = val.encode()
+        self.attr.pci_name = <char*>PyMem_Malloc(len(pci_name_bytes))
+        strcpy(<char*>self.attr.pci_name, pci_name_bytes)
+
+
+cdef class Mlx5VfioContext(Mlx5Context):
+    """
+    Mlx5VfioContext class is used to easily initialize and open a context over
+    a mlx5 vfio device.
+    It is initialized based on the passed mlx5 vfio attributes (Mlx5VfioAttr),
+    by getting the relevant vfio device and opening it (creating a context).
+    """
+    def __init__(self, Mlx5VfioAttr attr):
+        super(Context, self).__init__()
+        cdef v.ibv_device **dev_list
+
+        self.name = attr.pci_name
+        self.pds = weakref.WeakSet()
+        self.devx_umems = weakref.WeakSet()
+        self.devx_objs = weakref.WeakSet()
+        self.uars = weakref.WeakSet()
+
+        dev_list = dv.mlx5dv_get_vfio_device_list(&attr.attr)
+        if dev_list == NULL:
+            raise PyverbsRDMAErrno('Failed to get VFIO device list')
+        self.device = dev_list[0]
+        if self.device == NULL:
+            raise PyverbsRDMAError('Failed to get VFIO device')
+        try:
+            self.context = v.ibv_open_device(self.device)
+            if self.context == NULL:
+                raise PyverbsRDMAErrno('Failed to open mlx5 VFIO device '
+                                       f'({self.device.name.decode()})')
+        finally:
+            v.ibv_free_device_list(dev_list)
+
+    def get_events_fd(self):
+        """
+        Gets the file descriptor to manage driver events.
+        :return: The file descriptor to be used for managing driver events.
+        """
+        fd = dv.mlx5dv_vfio_get_events_fd(self.context)
+        if fd < 0:
+            raise PyverbsRDMAError('Failed to get VFIO events FD', -fd)
+        return fd
+
+    def process_events(self):
+        """
+        Process events on the vfio device.
+        This method should run from application thread to maintain device events.
+        :return: None
+        """
+        rc = dv.mlx5dv_vfio_process_events(self.context)
+        if rc:
+            raise PyverbsRDMAError('VFIO process events failed', rc)
+
+    cpdef close(self):
+        if self.context != NULL:
+            self.logger.debug('Closing Mlx5VfioContext')
+            close_weakrefs([self.pds, self.devx_objs, self.devx_umems, self.uars])
+            rc = v.ibv_close_device(self.context)
+            if rc != 0:
+                raise PyverbsRDMAErrno(f'Failed to close device {self.name}')
+            self.context = NULL
diff --git a/pyverbs/providers/mlx5/mlx5dv.pxd b/pyverbs/providers/mlx5/mlx5dv.pxd
index 968cbdb..490c697 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv.pxd
@@ -86,3 +86,6 @@ cdef class Mlx5DevxObj(PyverbsCM):
 
 cdef class Mlx5Cqe64(PyverbsObject):
     cdef dv.mlx5_cqe64 *cqe
+
+cdef class Mlx5VfioAttr(PyverbsObject):
+    cdef dv.mlx5dv_vfio_context_attr attr
diff --git a/pyverbs/providers/mlx5/mlx5dv_enums.pxd b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
index 60713e8..94599ab 100644
--- a/pyverbs/providers/mlx5/mlx5dv_enums.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
@@ -215,6 +215,9 @@ cdef extern from 'infiniband/mlx5dv.h':
         MLX5_SEND_WQE_BB
         MLX5_SEND_WQE_SHIFT
 
+    cpdef enum mlx5dv_vfio_context_attr_flags:
+        MLX5DV_VFIO_CTX_FLAGS_INIT_LINK_DOWN
+
     cpdef unsigned long long MLX5DV_RES_TYPE_QP
     cpdef unsigned long long MLX5DV_RES_TYPE_RWQ
     cpdef unsigned long long MLX5DV_RES_TYPE_DBR
-- 
1.8.3.1