From: Sagi Grimberg <sagi@grimberg.me>
To: linux-nvme@lists.infradead.org
Cc: Christoph Hellwig <hch@lst.de>,
Keith Busch <keith.busch@intel.com>,
linux-block@vger.kernel.org, Jens Axboe <axboe@kernel.dk>
Subject: [PATCH 5/5] nvme-rdma: support read/write queue separation
Date: Tue, 11 Dec 2018 02:49:35 -0800 [thread overview]
Message-ID: <20181211104936.25333-6-sagi@grimberg.me> (raw)
In-Reply-To: <20181211104936.25333-1-sagi@grimberg.me>
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
---
drivers/nvme/host/rdma.c | 39 ++++++++++++++++++++++++++++++++++++---
1 file changed, 36 insertions(+), 3 deletions(-)
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 5057d5ab5aaa..cfe823a491f2 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -645,6 +645,8 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
nr_io_queues = min_t(unsigned int, nr_io_queues,
ibdev->num_comp_vectors);
+ nr_io_queues += min(opts->nr_write_queues, num_online_cpus());
+
ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
if (ret)
return ret;
@@ -714,6 +716,7 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl,
set->driver_data = ctrl;
set->nr_hw_queues = nctrl->queue_count - 1;
set->timeout = NVME_IO_TIMEOUT;
+ set->nr_maps = 2 /* default + read */;
}
ret = blk_mq_alloc_tag_set(set);
@@ -1750,8 +1753,37 @@ static void nvme_rdma_complete_rq(struct request *rq)
static int nvme_rdma_map_queues(struct blk_mq_tag_set *set)
{
struct nvme_rdma_ctrl *ctrl = set->driver_data;
+ struct blk_mq_queue_map *map;
+ int offset = 0;
+
+ if (ctrl->ctrl.opts->nr_write_queues) {
+ /* separate read/write queues */
+ map = &set->map[HCTX_TYPE_DEFAULT];
+ map->queue_offset = offset;
+ map->nr_queues = ctrl->ctrl.opts->nr_write_queues;
+ blk_mq_rdma_map_queues(map, ctrl->device->dev, 0);
+ offset += map->nr_queues;
+
+ map = &set->map[HCTX_TYPE_READ];
+ map->nr_queues = ctrl->ctrl.opts->nr_io_queues;
+ map->queue_offset = offset;
+ blk_mq_rdma_map_queues(map, ctrl->device->dev, offset);
+ offset += map->nr_queues;
- return blk_mq_rdma_map_queues(&set->map[0], ctrl->device->dev, 0);
+ } else {
+ /* mixed read/write queues */
+ map = &set->map[HCTX_TYPE_DEFAULT];
+ map->queue_offset = 0;
+ map->nr_queues = ctrl->ctrl.opts->nr_io_queues;
+ blk_mq_rdma_map_queues(map, ctrl->device->dev, 0);
+
+ map = &set->map[HCTX_TYPE_READ];
+ map->queue_offset = 0;
+ map->nr_queues = ctrl->ctrl.opts->nr_io_queues;
+ blk_mq_rdma_map_queues(map, ctrl->device->dev, 0);
+ }
+
+ return 0;
}
static const struct blk_mq_ops nvme_rdma_mq_ops = {
@@ -1906,7 +1938,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
- ctrl->ctrl.queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
+ ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 1;
ctrl->ctrl.sqsize = opts->queue_size - 1;
ctrl->ctrl.kato = opts->kato;
@@ -1957,7 +1989,8 @@ static struct nvmf_transport_ops nvme_rdma_transport = {
.module = THIS_MODULE,
.required_opts = NVMF_OPT_TRADDR,
.allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
- NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO,
+ NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
+ NVMF_OPT_NR_IO_QUEUES,
.create_ctrl = nvme_rdma_create_ctrl,
};
--
2.17.1
next prev parent reply other threads:[~2018-12-11 10:50 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-12-11 10:49 [PATCH 0/5] implement nvmf read/write queue maps Sagi Grimberg
2018-12-11 10:49 ` [PATCH 1/5] blk-mq-rdma: pass in queue map to blk_mq_rdma_map_queues Sagi Grimberg
2018-12-11 13:34 ` Christoph Hellwig
2018-12-11 10:49 ` [PATCH 2/5] nvme-fabrics: add missing nvmf_ctrl_options documentation Sagi Grimberg
2018-12-11 13:35 ` Christoph Hellwig
2018-12-11 10:49 ` [PATCH 3/5] nvme-fabrics: allow user to set nr_write_queues for separate queue maps Sagi Grimberg
2018-12-11 13:35 ` Christoph Hellwig
2018-12-11 10:49 ` [PATCH 4/5] nvme-tcp: support separate queue maps for read and write Sagi Grimberg
2018-12-11 13:41 ` Christoph Hellwig
2018-12-11 23:11 ` Sagi Grimberg
2018-12-11 10:49 ` Sagi Grimberg [this message]
2018-12-11 13:42 ` [PATCH 5/5] nvme-rdma: support read/write queue separation Christoph Hellwig
2018-12-11 10:49 ` [PATCH nvme-cli 6/5] fabrics: pass in nr_write_queues Sagi Grimberg
2018-12-11 19:30 ` Keith Busch
2018-12-11 23:34 ` Sagi Grimberg
2018-12-11 13:28 ` [PATCH 0/5] implement nvmf read/write queue maps Christoph Hellwig
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20181211104936.25333-6-sagi@grimberg.me \
--to=sagi@grimberg.me \
--cc=axboe@kernel.dk \
--cc=hch@lst.de \
--cc=keith.busch@intel.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-nvme@lists.infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).