From: Dave Jiang <dave.jiang@intel.com>
To: vkoul@kernel.org, tglx@linutronix.de, mingo@redhat.com,
bp@alien8.de, hpa@zytor.com, bhelgaas@google.com,
gregkh@linuxfoundation.org, arnd@arndb.de
Cc: dmaengine@vger.kernel.org, linux-kernel@vger.kernel.org,
x86@kernel.org, dan.j.williams@intel.com, ashok.raj@intel.com,
fenghua.yu@intel.com, tony.luck@intel.com, jing.lin@intel.com,
sanjay.k.kumar@intel.com, dave.hansen@intel.com
Subject: [PATCH v2 5/9] dmaengine: idxd: move submission to sbitmap_queue
Date: Mon, 18 May 2020 11:53:51 -0700 [thread overview]
Message-ID: <158982803103.37989.17841991953662562685.stgit@djiang5-desk3.ch.intel.com> (raw)
In-Reply-To: <158982749959.37989.2096629611303670415.stgit@djiang5-desk3.ch.intel.com>
Kill the percpu-rwsem for work submission in favor of an sbitmap_queue.
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dma/Kconfig | 2 +
drivers/dma/idxd/device.c | 14 ++++-----
drivers/dma/idxd/idxd.h | 6 +---
drivers/dma/idxd/init.c | 20 ------------
drivers/dma/idxd/submit.c | 74 +++++++++++++++++++++++----------------------
5 files changed, 46 insertions(+), 70 deletions(-)
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 037ada9cc056..e817781b656a 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -284,8 +284,8 @@ config INTEL_IDMA64
config INTEL_IDXD
tristate "Intel Data Accelerators support"
depends on PCI && X86_64
+ depends on SBITMAP
select DMA_ENGINE
- select SBITMAP
help
Enable support for the Intel(R) data accelerators present
in Intel Xeon CPU.
diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index 8fe344412bd9..95b8820ef087 100644
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -160,16 +160,14 @@ static int alloc_descs(struct idxd_wq *wq, int num)
int idxd_wq_alloc_resources(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
- struct idxd_group *group = wq->group;
struct device *dev = &idxd->pdev->dev;
int rc, num_descs, i;
if (wq->type != IDXD_WQT_KERNEL)
return 0;
- num_descs = wq->size +
- idxd->hw.gen_cap.max_descs_per_engine * group->num_engines;
- wq->num_descs = num_descs;
+ wq->num_descs = wq->size;
+ num_descs = wq->size;
rc = alloc_hw_descs(wq, num_descs);
if (rc < 0)
@@ -187,8 +185,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
if (rc < 0)
goto fail_alloc_descs;
- rc = sbitmap_init_node(&wq->sbmap, num_descs, -1, GFP_KERNEL,
- dev_to_node(dev));
+ rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
+ dev_to_node(dev));
if (rc < 0)
goto fail_sbitmap_init;
@@ -201,7 +199,7 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
sizeof(struct dsa_completion_record) * i;
desc->id = i;
desc->wq = wq;
-
+ desc->cpu = -1;
dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan);
desc->txd.tx_submit = idxd_dma_tx_submit;
}
@@ -227,7 +225,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq)
free_hw_descs(wq);
free_descs(wq);
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
- sbitmap_free(&wq->sbmap);
+ sbitmap_queue_free(&wq->sbq);
}
int idxd_wq_enable(struct idxd_wq *wq)
diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index 013d68d42bc4..0769354df4cf 100644
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -104,7 +104,6 @@ struct idxd_wq {
enum idxd_wq_state state;
unsigned long flags;
union wqcfg wqcfg;
- atomic_t dq_count; /* dedicated queue flow control */
u32 vec_ptr; /* interrupt steering */
struct dsa_hw_desc **hw_descs;
int num_descs;
@@ -112,10 +111,8 @@ struct idxd_wq {
dma_addr_t compls_addr;
int compls_size;
struct idxd_desc **descs;
- struct sbitmap sbmap;
+ struct sbitmap_queue sbq;
struct dma_chan dma_chan;
- struct percpu_rw_semaphore submit_lock;
- wait_queue_head_t submit_waitq;
char name[WQ_NAME_SIZE + 1];
};
@@ -205,6 +202,7 @@ struct idxd_desc {
struct llist_node llnode;
struct list_head list;
int id;
+ int cpu;
struct idxd_wq *wq;
};
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index 51cdf77038dc..c7c61974f20f 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -141,17 +141,6 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
return rc;
}
-static void idxd_wqs_free_lock(struct idxd_device *idxd)
-{
- int i;
-
- for (i = 0; i < idxd->max_wqs; i++) {
- struct idxd_wq *wq = &idxd->wqs[i];
-
- percpu_free_rwsem(&wq->submit_lock);
- }
-}
-
static int idxd_setup_internals(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
@@ -182,19 +171,11 @@ static int idxd_setup_internals(struct idxd_device *idxd)
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
- int rc;
wq->id = i;
wq->idxd = idxd;
mutex_init(&wq->wq_lock);
- atomic_set(&wq->dq_count, 0);
- init_waitqueue_head(&wq->submit_waitq);
wq->idxd_cdev.minor = -1;
- rc = percpu_init_rwsem(&wq->submit_lock);
- if (rc < 0) {
- idxd_wqs_free_lock(idxd);
- return rc;
- }
}
for (i = 0; i < idxd->max_engines; i++) {
@@ -464,7 +445,6 @@ static void idxd_remove(struct pci_dev *pdev)
dev_dbg(&pdev->dev, "%s called\n", __func__);
idxd_cleanup_sysfs(idxd);
idxd_shutdown(pdev);
- idxd_wqs_free_lock(idxd);
mutex_lock(&idxd_idr_lock);
idr_remove(&idxd_idrs[idxd->type], idxd->id);
mutex_unlock(&idxd_idr_lock);
diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c
index 45a0c5869a0a..156a1ee233aa 100644
--- a/drivers/dma/idxd/submit.c
+++ b/drivers/dma/idxd/submit.c
@@ -8,61 +8,61 @@
#include "idxd.h"
#include "registers.h"
-struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype)
+static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
{
struct idxd_desc *desc;
- int idx;
+
+ desc = wq->descs[idx];
+ memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
+ memset(desc->completion, 0, sizeof(struct dsa_completion_record));
+ desc->cpu = cpu;
+ return desc;
+}
+
+struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype)
+{
+ int cpu, idx;
struct idxd_device *idxd = wq->idxd;
+ DEFINE_SBQ_WAIT(wait);
+ struct sbq_wait_state *ws;
+ struct sbitmap_queue *sbq;
if (idxd->state != IDXD_DEV_ENABLED)
return ERR_PTR(-EIO);
- if (optype == IDXD_OP_BLOCK)
- percpu_down_read(&wq->submit_lock);
- else if (!percpu_down_read_trylock(&wq->submit_lock))
- return ERR_PTR(-EBUSY);
-
- if (!atomic_add_unless(&wq->dq_count, 1, wq->size)) {
- int rc;
-
- if (optype == IDXD_OP_NONBLOCK) {
- percpu_up_read(&wq->submit_lock);
+ sbq = &wq->sbq;
+ idx = sbitmap_queue_get(sbq, &cpu);
+ if (idx < 0) {
+ if (optype == IDXD_OP_NONBLOCK)
return ERR_PTR(-EAGAIN);
- }
-
- percpu_up_read(&wq->submit_lock);
- percpu_down_write(&wq->submit_lock);
- rc = wait_event_interruptible(wq->submit_waitq,
- atomic_add_unless(&wq->dq_count,
- 1, wq->size) ||
- idxd->state != IDXD_DEV_ENABLED);
- percpu_up_write(&wq->submit_lock);
- if (rc < 0)
- return ERR_PTR(-EINTR);
- if (idxd->state != IDXD_DEV_ENABLED)
- return ERR_PTR(-EIO);
} else {
- percpu_up_read(&wq->submit_lock);
+ return __get_desc(wq, idx, cpu);
}
- idx = sbitmap_get(&wq->sbmap, 0, false);
- if (idx < 0) {
- atomic_dec(&wq->dq_count);
- return ERR_PTR(-EAGAIN);
+ ws = &sbq->ws[0];
+ for (;;) {
+ sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_INTERRUPTIBLE);
+ if (signal_pending_state(TASK_INTERRUPTIBLE, current))
+ break;
+ idx = sbitmap_queue_get(sbq, &cpu);
+ if (idx > 0)
+ break;
+ schedule();
}
- desc = wq->descs[idx];
- memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
- memset(desc->completion, 0, sizeof(struct dsa_completion_record));
- return desc;
+ sbitmap_finish_wait(sbq, ws, &wait);
+ if (idx < 0)
+ return ERR_PTR(-EAGAIN);
+
+ return __get_desc(wq, idx, cpu);
}
void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc)
{
- atomic_dec(&wq->dq_count);
+ int cpu = desc->cpu;
- sbitmap_clear_bit(&wq->sbmap, desc->id);
- wake_up(&wq->submit_waitq);
+ desc->cpu = -1;
+ sbitmap_queue_clear(&wq->sbq, desc->id, cpu);
}
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
next prev parent reply other threads:[~2020-05-18 18:53 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-18 18:53 [PATCH v2 0/9] Add shared workqueue support for idxd driver Dave Jiang
2020-05-18 18:53 ` [PATCH v2 1/9] x86/cpufeatures: Enumerate ENQCMD and ENQCMDS instructions Dave Jiang
2020-05-18 18:53 ` [PATCH v2 2/9] x86/asm: move the raw asm in iosubmit_cmds512() to special_insns.h Dave Jiang
2020-05-18 18:53 ` [PATCH v2 3/9] x86/asm: add enqcmds() to support ENQCMDS instruction Dave Jiang
2020-05-18 18:53 ` [PATCH v2 4/9] dmaengine: idxd: add work queue drain support Dave Jiang
2020-05-18 18:53 ` Dave Jiang [this message]
2020-05-18 18:53 ` [PATCH v2 6/9] dmaengine: idxd: add shared workqueue support Dave Jiang
2020-05-18 18:54 ` [PATCH v2 7/9] dmaengine: idxd: clean up descriptors with fault error Dave Jiang
2020-05-18 18:54 ` [PATCH v2 8/9] dmaengine: idxd: add leading / for sysfspath in ABI documentation Dave Jiang
2020-05-18 18:54 ` [PATCH v2 9/9] dmaengine: idxd: add ABI documentation for shared wq Dave Jiang
2020-06-01 22:09 ` [PATCH v2 0/9] Add shared workqueue support for idxd driver Dave Jiang
2020-06-03 6:33 ` Vinod Koul
2020-06-03 15:30 ` Dave Jiang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=158982803103.37989.17841991953662562685.stgit@djiang5-desk3.ch.intel.com \
--to=dave.jiang@intel.com \
--cc=arnd@arndb.de \
--cc=ashok.raj@intel.com \
--cc=bhelgaas@google.com \
--cc=bp@alien8.de \
--cc=dan.j.williams@intel.com \
--cc=dave.hansen@intel.com \
--cc=dmaengine@vger.kernel.org \
--cc=fenghua.yu@intel.com \
--cc=gregkh@linuxfoundation.org \
--cc=hpa@zytor.com \
--cc=jing.lin@intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=sanjay.k.kumar@intel.com \
--cc=tglx@linutronix.de \
--cc=tony.luck@intel.com \
--cc=vkoul@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).