From: Christoph Hellwig <hch@lst.de> To: Jens Axboe <axboe@fb.com>, Keith Busch <keith.busch@intel.com>, Sagi Grimberg <sagi@grimberg.me> Cc: Max Gurtovoy <maxg@mellanox.com>, linux-nvme@lists.infradead.org, linux-block@vger.kernel.org Subject: [PATCH 08/13] nvme-pci: remove the CQ lock for interrupt driven queues Date: Thu, 29 Nov 2018 20:13:05 +0100 [thread overview] Message-ID: <20181129191310.9795-9-hch@lst.de> (raw) In-Reply-To: <20181129191310.9795-1-hch@lst.de> Now that we can't poll regular, interrupt driven I/O queues there is almost nothing that can race with an interrupt. The only possible other contexts polling a CQ are the error handler and queue shutdown, and both are so far off in the slow path that we can simply use the big hammer of disabling interrupts. With that we can stop taking the cq_lock for normal queues. Signed-off-by: Christoph Hellwig <hch@lst.de> --- drivers/nvme/host/pci.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index fb8db7d8170a..d43925fba560 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -186,7 +186,8 @@ struct nvme_queue { struct nvme_dev *dev; spinlock_t sq_lock; struct nvme_command *sq_cmds; - spinlock_t cq_lock ____cacheline_aligned_in_smp; + /* only used for poll queues: */ + spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; volatile struct nvme_completion *cqes; struct blk_mq_tags **tags; dma_addr_t sq_dma_addr; @@ -1050,12 +1051,16 @@ static irqreturn_t nvme_irq(int irq, void *data) irqreturn_t ret = IRQ_NONE; u16 start, end; - spin_lock(&nvmeq->cq_lock); + /* + * The rmb/wmb pair ensures we see all updates from a previous run of + * the irq handler, even if that was on another CPU. + */ + rmb(); if (nvmeq->cq_head != nvmeq->last_cq_head) ret = IRQ_HANDLED; nvme_process_cq(nvmeq, &start, &end, -1); nvmeq->last_cq_head = nvmeq->cq_head; - spin_unlock(&nvmeq->cq_lock); + wmb(); if (start != end) { nvme_complete_cqes(nvmeq, start, end); @@ -1079,13 +1084,24 @@ static irqreturn_t nvme_irq_check(int irq, void *data) */ static int nvme_poll_irqdisable(struct nvme_queue *nvmeq, unsigned int tag) { - unsigned long flags; + struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); u16 start, end; int found; - spin_lock_irqsave(&nvmeq->cq_lock, flags); + /* + * For a poll queue we need to protect against the polling thread + * using the CQ lock. For normal interrupt driven threads we have + * to disable the interrupt to avoid racing with it. + */ + if (nvmeq->cq_vector == -1) + spin_lock(&nvmeq->cq_poll_lock); + else + disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); found = nvme_process_cq(nvmeq, &start, &end, tag); - spin_unlock_irqrestore(&nvmeq->cq_lock, flags); + if (nvmeq->cq_vector == -1) + spin_unlock(&nvmeq->cq_poll_lock); + else + enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); nvme_complete_cqes(nvmeq, start, end); return found; @@ -1100,9 +1116,9 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx) if (!nvme_cqe_pending(nvmeq)) return 0; - spin_lock(&nvmeq->cq_lock); + spin_lock(&nvmeq->cq_poll_lock); found = nvme_process_cq(nvmeq, &start, &end, -1); - spin_unlock(&nvmeq->cq_lock); + spin_unlock(&nvmeq->cq_poll_lock); nvme_complete_cqes(nvmeq, start, end); return found; @@ -1482,7 +1498,7 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) nvmeq->q_dmadev = dev->dev; nvmeq->dev = dev; spin_lock_init(&nvmeq->sq_lock); - spin_lock_init(&nvmeq->cq_lock); + spin_lock_init(&nvmeq->cq_poll_lock); nvmeq->cq_head = 0; nvmeq->cq_phase = 1; nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; @@ -1518,7 +1534,6 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) { struct nvme_dev *dev = nvmeq->dev; - spin_lock_irq(&nvmeq->cq_lock); nvmeq->sq_tail = 0; nvmeq->last_sq_tail = 0; nvmeq->cq_head = 0; @@ -1527,7 +1542,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); nvme_dbbuf_init(dev, nvmeq, qid); dev->online_queues++; - spin_unlock_irq(&nvmeq->cq_lock); + wmb(); /* ensure the first interrupt sees the initialization */ } static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) -- 2.19.1
WARNING: multiple messages have this Message-ID (diff)
From: hch@lst.de (Christoph Hellwig) Subject: [PATCH 08/13] nvme-pci: remove the CQ lock for interrupt driven queues Date: Thu, 29 Nov 2018 20:13:05 +0100 [thread overview] Message-ID: <20181129191310.9795-9-hch@lst.de> (raw) In-Reply-To: <20181129191310.9795-1-hch@lst.de> Now that we can't poll regular, interrupt driven I/O queues there is almost nothing that can race with an interrupt. The only possible other contexts polling a CQ are the error handler and queue shutdown, and both are so far off in the slow path that we can simply use the big hammer of disabling interrupts. With that we can stop taking the cq_lock for normal queues. Signed-off-by: Christoph Hellwig <hch at lst.de> --- drivers/nvme/host/pci.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index fb8db7d8170a..d43925fba560 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -186,7 +186,8 @@ struct nvme_queue { struct nvme_dev *dev; spinlock_t sq_lock; struct nvme_command *sq_cmds; - spinlock_t cq_lock ____cacheline_aligned_in_smp; + /* only used for poll queues: */ + spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; volatile struct nvme_completion *cqes; struct blk_mq_tags **tags; dma_addr_t sq_dma_addr; @@ -1050,12 +1051,16 @@ static irqreturn_t nvme_irq(int irq, void *data) irqreturn_t ret = IRQ_NONE; u16 start, end; - spin_lock(&nvmeq->cq_lock); + /* + * The rmb/wmb pair ensures we see all updates from a previous run of + * the irq handler, even if that was on another CPU. + */ + rmb(); if (nvmeq->cq_head != nvmeq->last_cq_head) ret = IRQ_HANDLED; nvme_process_cq(nvmeq, &start, &end, -1); nvmeq->last_cq_head = nvmeq->cq_head; - spin_unlock(&nvmeq->cq_lock); + wmb(); if (start != end) { nvme_complete_cqes(nvmeq, start, end); @@ -1079,13 +1084,24 @@ static irqreturn_t nvme_irq_check(int irq, void *data) */ static int nvme_poll_irqdisable(struct nvme_queue *nvmeq, unsigned int tag) { - unsigned long flags; + struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); u16 start, end; int found; - spin_lock_irqsave(&nvmeq->cq_lock, flags); + /* + * For a poll queue we need to protect against the polling thread + * using the CQ lock. For normal interrupt driven threads we have + * to disable the interrupt to avoid racing with it. + */ + if (nvmeq->cq_vector == -1) + spin_lock(&nvmeq->cq_poll_lock); + else + disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); found = nvme_process_cq(nvmeq, &start, &end, tag); - spin_unlock_irqrestore(&nvmeq->cq_lock, flags); + if (nvmeq->cq_vector == -1) + spin_unlock(&nvmeq->cq_poll_lock); + else + enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); nvme_complete_cqes(nvmeq, start, end); return found; @@ -1100,9 +1116,9 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx) if (!nvme_cqe_pending(nvmeq)) return 0; - spin_lock(&nvmeq->cq_lock); + spin_lock(&nvmeq->cq_poll_lock); found = nvme_process_cq(nvmeq, &start, &end, -1); - spin_unlock(&nvmeq->cq_lock); + spin_unlock(&nvmeq->cq_poll_lock); nvme_complete_cqes(nvmeq, start, end); return found; @@ -1482,7 +1498,7 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) nvmeq->q_dmadev = dev->dev; nvmeq->dev = dev; spin_lock_init(&nvmeq->sq_lock); - spin_lock_init(&nvmeq->cq_lock); + spin_lock_init(&nvmeq->cq_poll_lock); nvmeq->cq_head = 0; nvmeq->cq_phase = 1; nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; @@ -1518,7 +1534,6 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) { struct nvme_dev *dev = nvmeq->dev; - spin_lock_irq(&nvmeq->cq_lock); nvmeq->sq_tail = 0; nvmeq->last_sq_tail = 0; nvmeq->cq_head = 0; @@ -1527,7 +1542,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); nvme_dbbuf_init(dev, nvmeq, qid); dev->online_queues++; - spin_unlock_irq(&nvmeq->cq_lock); + wmb(); /* ensure the first interrupt sees the initialization */ } static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) -- 2.19.1
next prev parent reply other threads:[~2018-11-29 19:13 UTC|newest] Thread overview: 68+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-11-29 19:12 block and nvme polling improvements V2 Christoph Hellwig 2018-11-29 19:12 ` Christoph Hellwig 2018-11-29 19:12 ` [PATCH 01/13] block: move queues types to the block layer Christoph Hellwig 2018-11-29 19:12 ` Christoph Hellwig 2018-11-29 19:50 ` Jens Axboe 2018-11-29 19:50 ` Jens Axboe 2018-11-30 7:56 ` Christoph Hellwig 2018-11-30 7:56 ` Christoph Hellwig 2018-11-30 15:20 ` Jens Axboe 2018-11-30 15:20 ` Jens Axboe 2018-11-30 15:21 ` Christoph Hellwig 2018-11-30 15:21 ` Christoph Hellwig 2018-11-29 20:19 ` Keith Busch 2018-11-29 20:19 ` Keith Busch 2018-11-29 20:25 ` Jens Axboe 2018-11-29 20:25 ` Jens Axboe 2018-11-30 8:00 ` Christoph Hellwig 2018-11-30 8:00 ` Christoph Hellwig 2018-11-30 14:40 ` Keith Busch 2018-11-30 14:40 ` Keith Busch 2018-11-30 15:20 ` Jens Axboe 2018-11-30 15:20 ` Jens Axboe 2018-11-29 19:12 ` [PATCH 02/13] nvme-pci: use atomic bitops to mark a queue enabled Christoph Hellwig 2018-11-29 19:12 ` Christoph Hellwig 2018-11-29 20:19 ` Keith Busch 2018-11-29 20:19 ` Keith Busch 2018-11-29 19:13 ` [PATCH 03/13] nvme-pci: cleanup SQ allocation a bit Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 20:22 ` Keith Busch 2018-11-29 20:22 ` Keith Busch 2018-11-29 19:13 ` [PATCH 04/13] nvme-pci: only allow polling with separate poll queues Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 05/13] nvme-pci: consolidate code for polling non-dedicated queues Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 06/13] nvme-pci: refactor nvme_disable_io_queues Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 20:37 ` Keith Busch 2018-11-29 20:37 ` Keith Busch 2018-11-29 19:13 ` [PATCH 07/13] nvme-pci: don't poll from irq context when deleting queues Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 20:36 ` Keith Busch 2018-11-29 20:36 ` Keith Busch 2018-11-30 8:08 ` Christoph Hellwig 2018-11-30 8:08 ` Christoph Hellwig 2018-11-30 14:45 ` Keith Busch 2018-11-30 14:45 ` Keith Busch 2018-11-29 19:13 ` Christoph Hellwig [this message] 2018-11-29 19:13 ` [PATCH 08/13] nvme-pci: remove the CQ lock for interrupt driven queues Christoph Hellwig 2018-11-29 21:08 ` Keith Busch 2018-11-29 21:08 ` Keith Busch 2018-11-30 8:16 ` Christoph Hellwig 2018-11-30 8:16 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 09/13] nvme-rdma: remove I/O polling support Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 10/13] nvme-mpath: " Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 11/13] block: remove ->poll_fn Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 12/13] block: only allow polling if a poll queue_map exists Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig 2018-11-29 19:13 ` [PATCH 13/13] block: enable polling by default if a poll map is initalized Christoph Hellwig 2018-11-29 19:13 ` Christoph Hellwig -- strict thread matches above, loose matches on Subject: below -- 2018-12-02 16:46 block and nvme polling improvements V3 Christoph Hellwig 2018-12-02 16:46 ` [PATCH 08/13] nvme-pci: remove the CQ lock for interrupt driven queues Christoph Hellwig 2018-12-02 16:46 ` Christoph Hellwig 2018-12-04 1:08 ` Sagi Grimberg 2018-12-04 1:08 ` Sagi Grimberg 2018-11-21 16:23 block and nvme polling improvements Christoph Hellwig 2018-11-21 16:23 ` [PATCH 08/13] nvme-pci: remove the CQ lock for interrupt driven queues Christoph Hellwig 2018-11-21 16:23 ` Christoph Hellwig
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20181129191310.9795-9-hch@lst.de \ --to=hch@lst.de \ --cc=axboe@fb.com \ --cc=keith.busch@intel.com \ --cc=linux-block@vger.kernel.org \ --cc=linux-nvme@lists.infradead.org \ --cc=maxg@mellanox.com \ --cc=sagi@grimberg.me \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.