All of lore.kernel.org
 help / color / mirror / Atom feed
From: Dave Jiang <dave.jiang@intel.com>
To: vkoul@kernel.org
Cc: Dan Williams <dan.j.williams@intel.com>,
	dmaengine@vger.kernel.org, jgg@nvidia.com,
	ramesh.thomas@intel.com
Subject: [PATCH 15/18] dmaengine: idxd: create dmaengine driver for wq 'device'
Date: Fri, 21 May 2021 15:22:52 -0700	[thread overview]
Message-ID: <162163577286.260470.5823300826030028426.stgit@djiang5-desk3.ch.intel.com> (raw)
In-Reply-To: <162163546245.260470.18336189072934823712.stgit@djiang5-desk3.ch.intel.com>

The original architecture of /sys/bus/dsa invented a scheme whereby a
single entry in the list of bus drivers, /sys/bus/drivers/dsa, handled
all device types and internally routed them to different drivers.
Those internal drivers were invisible to userspace. Now, as
/sys/bus/dsa wants to grow support for alternate drivers for a given
device, for example vfio-mdev instead of kernel-internal-dmaengine, a
proper bus device-driver model is needed. The first step in that process
is separating the existing omnibus/implicit "dsa" driver into proper
individual drivers registered on /sys/bus/dsa. Establish the
idxd_dmaengine_drv driver that controls the enabling and disabling of the
wq and also register and unregister the dma channel.

idxd_wq_alloc_resources() and idxd_wq_free_resources() also get moved to
the dmaengine driver. The resources (dma descriptors allocation and setup)
are only used by the dmaengine driver and should only happen when it loads.

The char dev driver (cdev) related bits are left in the __drv_enable_wq()
and __drv_disable_wq() calls to be moved when we split out the char dev
driver just like how the dmaengine driver is split out.

WQ autoload support is not expected currently. With the amount of
configuration needed for the device, the wq is always expected to
be enabled by a tool (or via sysfs) rather than auto enabled at driver
load.

Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 drivers/dma/idxd/device.c |   40 +++--------------------
 drivers/dma/idxd/dma.c    |   77 +++++++++++++++++++++++++++++++++++++++++++++
 drivers/dma/idxd/idxd.h   |    3 ++
 drivers/dma/idxd/init.c   |    7 ++++
 4 files changed, 92 insertions(+), 35 deletions(-)

diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index f1a82d46245f..85ee5d65c724 100644
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -1087,7 +1087,7 @@ int idxd_device_load_config(struct idxd_device *idxd)
 	return 0;
 }
 
-static int __drv_enable_wq(struct idxd_wq *wq)
+int __drv_enable_wq(struct idxd_wq *wq)
 {
 	struct idxd_device *idxd = wq->idxd;
 	struct device *dev = &idxd->pdev->dev;
@@ -1135,12 +1135,7 @@ static int __drv_enable_wq(struct idxd_wq *wq)
 		}
 	}
 
-	rc = idxd_wq_alloc_resources(wq);
-	if (rc < 0) {
-		dev_dbg(dev, "wq resource alloc failed\n");
-		goto err;
-	}
-
+	rc = 0;
 	spin_lock_irqsave(&idxd->dev_lock, flags);
 	if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
 		rc = idxd_device_config(idxd);
@@ -1164,21 +1159,7 @@ static int __drv_enable_wq(struct idxd_wq *wq)
 
 	wq->client_count = 0;
 
-	if (wq->type == IDXD_WQT_KERNEL) {
-		rc = idxd_wq_init_percpu_ref(wq);
-		if (rc < 0) {
-			dev_dbg(dev, "wq %d percpu_ref setup failed\n", wq->id);
-			goto err_cpu_ref;
-		}
-	}
-
-	if (is_idxd_wq_dmaengine(wq)) {
-		rc = idxd_register_dma_channel(wq);
-		if (rc < 0) {
-			dev_dbg(dev, "wq %d DMA channel register failed\n", wq->id);
-			goto err_client;
-		}
-	} else if (is_idxd_wq_cdev(wq)) {
+	if (is_idxd_wq_cdev(wq)) {
 		rc = idxd_wq_add_cdev(wq);
 		if (rc < 0) {
 			dev_dbg(dev, "wq %d cdev creation failed\n", wq->id);
@@ -1186,12 +1167,9 @@ static int __drv_enable_wq(struct idxd_wq *wq)
 		}
 	}
 
-	dev_info(dev, "wq %s enabled\n", dev_name(wq_confdev(wq)));
 	return 0;
 
 err_client:
-	idxd_wq_quiesce(wq);
-err_cpu_ref:
 	idxd_wq_unmap_portal(wq);
 err_map_portal:
 	rc = idxd_wq_disable(wq);
@@ -1211,19 +1189,14 @@ int drv_enable_wq(struct idxd_wq *wq)
 	return rc;
 }
 
-static void __drv_disable_wq(struct idxd_wq *wq)
+void __drv_disable_wq(struct idxd_wq *wq)
 {
 	struct idxd_device *idxd = wq->idxd;
 	struct device *dev = &idxd->pdev->dev;
 
 	lockdep_assert_held(&wq->wq_lock);
 
-	if (wq->type == IDXD_WQT_KERNEL)
-		idxd_wq_quiesce(wq);
-
-	if (is_idxd_wq_dmaengine(wq))
-		idxd_unregister_dma_channel(wq);
-	else if (is_idxd_wq_cdev(wq))
+	if (is_idxd_wq_cdev(wq))
 		idxd_wq_del_cdev(wq);
 
 	if (idxd_wq_refcount(wq))
@@ -1235,10 +1208,7 @@ static void __drv_disable_wq(struct idxd_wq *wq)
 	idxd_wq_drain(wq);
 	idxd_wq_reset(wq);
 
-	idxd_wq_free_resources(wq);
 	wq->client_count = 0;
-
-	dev_info(dev, "wq %s disabled\n", dev_name(wq_confdev(wq)));
 }
 
 void drv_disable_wq(struct idxd_wq *wq)
diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c
index 2e52f9a50519..7e3281700183 100644
--- a/drivers/dma/idxd/dma.c
+++ b/drivers/dma/idxd/dma.c
@@ -262,3 +262,80 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq)
 	wq->idxd_chan = NULL;
 	put_device(wq_confdev(wq));
 }
+
+static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
+{
+	struct device *dev = &idxd_dev->conf_dev;
+	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
+	struct idxd_device *idxd = wq->idxd;
+	int rc;
+
+	if (idxd->state != IDXD_DEV_ENABLED)
+		return -ENXIO;
+
+	mutex_lock(&wq->wq_lock);
+	wq->type = IDXD_WQT_KERNEL;
+	rc = __drv_enable_wq(wq);
+	if (rc < 0) {
+		dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
+		rc = -ENXIO;
+		goto err;
+	}
+
+	rc = idxd_wq_alloc_resources(wq);
+	if (rc < 0) {
+		dev_dbg(dev, "WQ resource alloc failed\n");
+		goto err_res_alloc;
+	}
+
+	rc = idxd_wq_init_percpu_ref(wq);
+	if (rc < 0) {
+		dev_dbg(dev, "percpu_ref setup failed\n");
+		goto err_ref;
+	}
+
+	rc = idxd_register_dma_channel(wq);
+	if (rc < 0) {
+		dev_dbg(dev, "Failed to register dma channel\n");
+		goto err_dma;
+	}
+
+	mutex_unlock(&wq->wq_lock);
+	return 0;
+
+err_dma:
+	idxd_wq_quiesce(wq);
+err_ref:
+	idxd_wq_free_resources(wq);
+err_res_alloc:
+	__drv_disable_wq(wq);
+err:
+	wq->type = IDXD_WQT_NONE;
+	mutex_unlock(&wq->wq_lock);
+	return rc;
+}
+
+static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
+{
+	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
+
+	mutex_lock(&wq->wq_lock);
+	idxd_wq_quiesce(wq);
+	idxd_unregister_dma_channel(wq);
+	__drv_disable_wq(wq);
+	idxd_wq_free_resources(wq);
+	wq->type = IDXD_WQT_NONE;
+	mutex_unlock(&wq->wq_lock);
+}
+
+static enum idxd_dev_type dev_types[] = {
+	IDXD_DEV_WQ,
+	IDXD_DEV_NONE,
+};
+
+struct idxd_device_driver idxd_dmaengine_drv = {
+	.probe = idxd_dmaengine_drv_probe,
+	.remove = idxd_dmaengine_drv_remove,
+	.name = "dmaengine",
+	.type = dev_types,
+};
diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index bb7561948abb..43ec1472ce25 100644
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -59,6 +59,7 @@ struct idxd_device_driver {
 
 extern struct idxd_device_driver dsa_drv;
 extern struct idxd_device_driver idxd_drv;
+extern struct idxd_device_driver idxd_dmaengine_drv;
 
 struct idxd_irq_entry {
 	struct idxd_device *idxd;
@@ -501,7 +502,9 @@ void idxd_unregister_idxd_drv(void);
 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
 int drv_enable_wq(struct idxd_wq *wq);
+int __drv_enable_wq(struct idxd_wq *wq);
 void drv_disable_wq(struct idxd_wq *wq);
+void __drv_disable_wq(struct idxd_wq *wq);
 int idxd_device_init_reset(struct idxd_device *idxd);
 int idxd_device_enable(struct idxd_device *idxd);
 int idxd_device_disable(struct idxd_device *idxd);
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index 01720d259259..72f49e072e1c 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -785,6 +785,10 @@ static int __init idxd_init_module(void)
 	if (err < 0)
 		goto err_idxd_driver_register;
 
+	err = idxd_driver_register(&idxd_dmaengine_drv);
+	if (err < 0)
+		goto err_idxd_dmaengine_driver_register;
+
 	err = idxd_driver_register(&dsa_drv);
 	if (err < 0)
 		goto err_dsa_driver_register;
@@ -804,6 +808,8 @@ static int __init idxd_init_module(void)
 err_cdev_register:
 	idxd_driver_unregister(&dsa_drv);
 err_dsa_driver_register:
+	idxd_driver_unregister(&idxd_dmaengine_drv);
+err_idxd_dmaengine_driver_register:
 	idxd_driver_unregister(&idxd_drv);
 err_idxd_driver_register:
 	idxd_unregister_bus_type();
@@ -813,6 +819,7 @@ module_init(idxd_init_module);
 
 static void __exit idxd_exit_module(void)
 {
+	idxd_driver_unregister(&idxd_dmaengine_drv);
 	idxd_driver_unregister(&idxd_drv);
 	idxd_driver_unregister(&dsa_drv);
 	pci_unregister_driver(&idxd_pci_driver);



  parent reply	other threads:[~2021-05-21 22:22 UTC|newest]

Thread overview: 24+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-21 22:21 [PATCH 00/18] Fix idxd sub-drivers setup Dave Jiang
2021-05-21 22:21 ` [PATCH 01/18] dmaengine: idxd: add driver register helper Dave Jiang
2021-05-21 22:21 ` [PATCH 02/18] dmaengine: idxd: add driver name Dave Jiang
2021-05-24  0:05   ` Jason Gunthorpe
2021-05-21 22:21 ` [PATCH 03/18] dmaengine: idxd: add 'struct idxd_dev' as wrapper for conf_dev Dave Jiang
2021-05-21 22:21 ` [PATCH 04/18] dmaengine: idxd: remove IDXD_DEV_CONF_READY Dave Jiang
2021-05-21 22:21 ` [PATCH 05/18] dmaengine: idxd: move wq_enable() to device.c Dave Jiang
2021-05-21 22:21 ` [PATCH 06/18] dmaengine: idxd: move wq_disable() " Dave Jiang
2021-05-21 22:22 ` [PATCH 07/18] dmaengine: idxd: remove bus shutdown Dave Jiang
2021-05-21 22:22 ` [PATCH 08/18] dmaengine: idxd: remove iax_bus_type prototype Dave Jiang
2021-05-21 22:22 ` [PATCH 09/18] dmaengine: idxd: fix bus_probe() and bus_remove() for dsa_bus Dave Jiang
2021-05-21 22:22 ` [PATCH 10/18] dmaengine: idxd: move probe() bits for idxd 'struct device' to device.c Dave Jiang
2021-05-22 19:38   ` kernel test robot
2021-05-21 22:22 ` [PATCH 11/18] dmaengine: idxd: idxd: move remove() " Dave Jiang
2021-05-21 22:22 ` [PATCH 12/18] dmanegine: idxd: open code the dsa_drv registration Dave Jiang
2021-05-21 22:22 ` [PATCH 13/18] dmaengine: idxd: add type to driver in order to allow device matching Dave Jiang
2021-05-21 22:22 ` [PATCH 14/18] dmaengine: idxd: create idxd_device sub-driver Dave Jiang
2021-05-21 22:22 ` Dave Jiang [this message]
2021-05-21 22:22 ` [PATCH 16/18] dmaengine: idxd: create user driver for wq 'device' Dave Jiang
2021-05-21 22:23 ` [PATCH 17/18] dmaengine: dsa: move dsa_bus_type out of idxd driver to standalone Dave Jiang
2021-05-21 22:23 ` [PATCH 18/18] dmaengine: idxd: move dsa_drv support to compatible mode Dave Jiang
2021-06-14 17:18 ` [PATCH 00/18] Fix idxd sub-drivers setup Dave Jiang
2021-06-28 16:53   ` Dave Jiang
2021-07-02  3:44     ` Vinod Koul

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=162163577286.260470.5823300826030028426.stgit@djiang5-desk3.ch.intel.com \
    --to=dave.jiang@intel.com \
    --cc=dan.j.williams@intel.com \
    --cc=dmaengine@vger.kernel.org \
    --cc=jgg@nvidia.com \
    --cc=ramesh.thomas@intel.com \
    --cc=vkoul@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.