kvm.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Max Gurtovoy <mgurtovoy@nvidia.com>
To: <alex.williamson@redhat.com>, <cohuck@redhat.com>,
	<kvm@vger.kernel.org>, <linux-kernel@vger.kernel.org>,
	<jgg@nvidia.com>
Cc: <aviadye@nvidia.com>, <oren@nvidia.com>, <shahafs@nvidia.com>,
	<parav@nvidia.com>, <artemp@nvidia.com>, <kwankhede@nvidia.com>,
	<ACurrid@nvidia.com>, <cjia@nvidia.com>, <yishaih@nvidia.com>,
	<kevin.tian@intel.com>, <hch@infradead.org>,
	<targupta@nvidia.com>, <shameerali.kolothum.thodi@huawei.com>,
	<liulongfang@huawei.com>, <yan.y.zhao@intel.com>,
	Max Gurtovoy <mgurtovoy@nvidia.com>
Subject: [PATCH 11/11] mlx5-vfio-pci: add new vfio_pci driver for mlx5 devices
Date: Thu, 3 Jun 2021 19:08:09 +0300	[thread overview]
Message-ID: <20210603160809.15845-12-mgurtovoy@nvidia.com> (raw)
In-Reply-To: <20210603160809.15845-1-mgurtovoy@nvidia.com>

This is the initial step for adding vendor specific vfio_pci driver for
mlx5 devices. mlx5_vfio_pci use vfio_pci_core to register to the VFIO
subsystem and also to implement the basic functionality of a pci device.

It will also extend this basic functionality and add mlx5 specific logic
for various features (such as live migration, for example).

Note: Although we've created the mlx5-vfio-pci.ko, the binding to
vfio-pci.ko will still work as before. It's fully backward compatible.
Of course, the extended vendor functionality will not exist in case one
will bind the device to the generic vfio_pci.ko.

Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
---
 drivers/vfio/pci/Kconfig         |   9 +++
 drivers/vfio/pci/Makefile        |   3 +
 drivers/vfio/pci/mlx5_vfio_pci.c | 130 +++++++++++++++++++++++++++++++
 3 files changed, 142 insertions(+)
 create mode 100644 drivers/vfio/pci/mlx5_vfio_pci.c

diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig
index 384d06661f30..9cdef46dd299 100644
--- a/drivers/vfio/pci/Kconfig
+++ b/drivers/vfio/pci/Kconfig
@@ -48,3 +48,12 @@ config VFIO_PCI_IGD
 	  and LPC bridge config space.
 
 	  To enable Intel IGD assignment through vfio-pci, say Y.
+
+config MLX5_VFIO_PCI
+	tristate "VFIO support for MLX5 PCI devices"
+	depends on VFIO_PCI_CORE && MLX5_CORE
+	help
+	  This provides a generic PCI support for MLX5 devices using the VFIO
+	  framework.
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile
index ddba4759cde7..a0df9c2a4bd9 100644
--- a/drivers/vfio/pci/Makefile
+++ b/drivers/vfio/pci/Makefile
@@ -2,9 +2,12 @@
 
 obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o
 obj-$(CONFIG_VFIO_PCI) += vfio-pci.o
+obj-$(CONFIG_MLX5_VFIO_PCI) += mlx5-vfio-pci.o
 
 vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o
 vfio-pci-core-$(CONFIG_S390) += vfio_pci_zdev.o
 
 vfio-pci-y := vfio_pci.o
 vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o
+
+mlx5-vfio-pci-y := mlx5_vfio_pci.o
diff --git a/drivers/vfio/pci/mlx5_vfio_pci.c b/drivers/vfio/pci/mlx5_vfio_pci.c
new file mode 100644
index 000000000000..0153682a3d3f
--- /dev/null
+++ b/drivers/vfio/pci/mlx5_vfio_pci.c
@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021, Mellanox Technologies. All rights reserved.
+ *     Author: Max Gurtovoy <mgurtovoy@nvidia.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/device.h>
+#include <linux/eventfd.h>
+#include <linux/file.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <linux/sched/mm.h>
+#include <linux/mlx5/driver.h>
+
+#include <linux/vfio_pci_core.h>
+
+static int mlx5_vfio_pci_open(struct vfio_device *core_vdev)
+{
+	struct vfio_pci_core_device *vdev =
+		container_of(core_vdev, struct vfio_pci_core_device, vdev);
+	int ret;
+
+	lockdep_assert_held(&core_vdev->reflck->lock);
+
+	ret = vfio_pci_core_enable(vdev);
+	if (ret)
+		return ret;
+
+	/* TODO: register migration region here for capable devices */
+
+	vfio_pci_core_finish_enable(vdev);
+
+	return 0;
+}
+
+static const struct vfio_device_ops mlx5_vfio_pci_ops = {
+	.name		= "mlx5-vfio-pci",
+	.open		= mlx5_vfio_pci_open,
+	.release	= vfio_pci_core_release,
+	.ioctl		= vfio_pci_core_ioctl,
+	.read		= vfio_pci_core_read,
+	.write		= vfio_pci_core_write,
+	.mmap		= vfio_pci_core_mmap,
+	.request	= vfio_pci_core_request,
+	.match		= vfio_pci_core_match,
+	.reflck_attach	= vfio_pci_core_reflck_attach,
+};
+
+static int mlx5_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct vfio_pci_core_device *vdev;
+	int ret;
+
+	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
+	if (!vdev)
+		return -ENOMEM;
+
+	ret = vfio_pci_core_register_device(vdev, pdev, &mlx5_vfio_pci_ops);
+	if (ret)
+		goto out_free;
+
+	dev_set_drvdata(&pdev->dev, vdev);
+	return 0;
+
+out_free:
+	kfree(vdev);
+	return ret;
+}
+
+static void mlx5_vfio_pci_remove(struct pci_dev *pdev)
+{
+	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
+	vfio_pci_core_unregister_device(vdev);
+	kfree(vdev);
+}
+
+static const struct pci_device_id mlx5_vfio_pci_table[] = {
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x101b) }, /* ConnectX-6 */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x101c) }, /* ConnectX-6 VF */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x101d) }, /* ConnectX-6 Dx */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x101e) }, /* ConnectX Family mlx5Gen Virtual Function */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x101f) }, /* ConnectX-6 LX */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0x1021) }, /* ConnectX-7 */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0xa2d3) }, /* BlueField integrated ConnectX-5 network controller VF */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, mlx5_vfio_pci_table);
+
+static struct pci_driver mlx5_vfio_pci_driver = {
+	.name			= "mlx5-vfio-pci",
+	.id_table		= mlx5_vfio_pci_table,
+	.probe			= mlx5_vfio_pci_probe,
+	.remove			= mlx5_vfio_pci_remove,
+#ifdef CONFIG_PCI_IOV
+	.sriov_configure	= vfio_pci_core_sriov_configure,
+#endif
+	.err_handler		= &vfio_pci_core_err_handlers,
+};
+
+static void __exit mlx5_vfio_pci_cleanup(void)
+{
+	pci_unregister_driver(&mlx5_vfio_pci_driver);
+}
+
+static int __init mlx5_vfio_pci_init(void)
+{
+	return pci_register_driver(&mlx5_vfio_pci_driver);
+}
+
+module_init(mlx5_vfio_pci_init);
+module_exit(mlx5_vfio_pci_cleanup);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Max Gurtovoy <mgurtovoy@nvidia.com>");
+MODULE_DESCRIPTION("MLX5 VFIO PCI - User Level meta-driver for MLX5 device family");
-- 
2.21.0


  parent reply	other threads:[~2021-06-03 16:09 UTC|newest]

Thread overview: 46+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-06-03 16:07 [RFC PATCH v4 00/11] Introduce vfio-pci-core subsystem Max Gurtovoy
2021-06-03 16:07 ` [PATCH 01/11] vfio-pci: rename vfio_pci.c to vfio_pci_core.c Max Gurtovoy
2021-06-03 16:08 ` [PATCH 02/11] vfio-pci: rename vfio_pci_private.h to vfio_pci_core.h Max Gurtovoy
2021-06-03 16:08 ` [PATCH 03/11] vfio-pci: rename vfio_pci_device to vfio_pci_core_device Max Gurtovoy
2021-06-03 16:08 ` [PATCH 04/11] vfio-pci: rename ops functions to fit core namings Max Gurtovoy
2021-06-03 16:08 ` [PATCH 05/11] vfio-pci: include vfio header in vfio_pci_core.h Max Gurtovoy
2021-06-03 16:08 ` [PATCH 06/11] vfio-pci: introduce vfio_pci.c Max Gurtovoy
2021-06-03 16:08 ` [PATCH 07/11] vfio-pci: move igd initialization to vfio_pci.c Max Gurtovoy
2021-06-03 16:08 ` [PATCH 08/11] PCI: add flags field to pci_device_id structure Max Gurtovoy
2021-06-03 16:08 ` [PATCH 09/11] PCI: add matching checks for driver_override binding Max Gurtovoy
2021-06-08 21:26   ` Alex Williamson
2021-06-08 22:45     ` Jason Gunthorpe
2021-06-09  1:27       ` Alex Williamson
2021-06-09  9:26         ` Max Gurtovoy
2021-06-13  8:19         ` Max Gurtovoy
2021-06-14  5:40           ` Christoph Hellwig
2021-06-14  8:18             ` Max Gurtovoy
2021-06-14 15:27               ` Christoph Hellwig
2021-06-14 16:01                 ` Jason Gunthorpe
2021-06-14 16:15                   ` Christoph Hellwig
2021-06-14 16:33                     ` Jason Gunthorpe
2021-06-14 18:42           ` Alex Williamson
2021-06-14 23:12             ` Max Gurtovoy
2021-06-15 15:00               ` Alex Williamson
2021-06-15 15:04                 ` Jason Gunthorpe
2021-06-15 16:20                   ` Alex Williamson
2021-06-15 20:42                     ` Jason Gunthorpe
2021-06-15 21:59                       ` Alex Williamson
2021-06-15 23:00                         ` Jason Gunthorpe
2021-06-15 23:22                           ` Alex Williamson
2021-06-15 23:32                             ` Jason Gunthorpe
2021-06-16  0:22                               ` Alex Williamson
2021-06-16  0:34                                 ` Jason Gunthorpe
2021-06-16 23:28                                   ` Max Gurtovoy
2021-06-16 23:33                                     ` Jason Gunthorpe
2021-06-16 23:42                                       ` Max Gurtovoy
2021-06-16 23:44                                         ` Jason Gunthorpe
2021-06-16 23:51                                           ` Max Gurtovoy
2021-06-16 23:56                                             ` Jason Gunthorpe
2021-06-20 14:46                                               ` Max Gurtovoy
2021-06-03 16:08 ` [PATCH 10/11] vfio-pci: introduce vfio_pci_core subsystem driver Max Gurtovoy
2021-06-08 21:26   ` Alex Williamson
2021-06-09  9:29     ` Max Gurtovoy
2021-06-03 16:08 ` Max Gurtovoy [this message]
2021-07-30  7:53 ` [RFC PATCH v4 00/11] Introduce vfio-pci-core subsystem Shameerali Kolothum Thodi
2021-07-30 11:55   ` Jason Gunthorpe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210603160809.15845-12-mgurtovoy@nvidia.com \
    --to=mgurtovoy@nvidia.com \
    --cc=ACurrid@nvidia.com \
    --cc=alex.williamson@redhat.com \
    --cc=artemp@nvidia.com \
    --cc=aviadye@nvidia.com \
    --cc=cjia@nvidia.com \
    --cc=cohuck@redhat.com \
    --cc=hch@infradead.org \
    --cc=jgg@nvidia.com \
    --cc=kevin.tian@intel.com \
    --cc=kvm@vger.kernel.org \
    --cc=kwankhede@nvidia.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=liulongfang@huawei.com \
    --cc=oren@nvidia.com \
    --cc=parav@nvidia.com \
    --cc=shahafs@nvidia.com \
    --cc=shameerali.kolothum.thodi@huawei.com \
    --cc=targupta@nvidia.com \
    --cc=yan.y.zhao@intel.com \
    --cc=yishaih@nvidia.com \
    --subject='Re: [PATCH 11/11] mlx5-vfio-pci: add new vfio_pci driver for mlx5 devices' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
on how to clone and mirror all data and code used for this inbox