From: Max Gurtovoy <maxg@mellanox.com> To: maxg@mellanox.com, linux-nvme@lists.infradead.org, kbusch@kernel.org, hch@lst.de, sagi@grimberg.me, martin.petersen@oracle.com, jsmart2021@gmail.com, axboe@kernel.dk Cc: vladimirk@mellanox.com, shlomin@mellanox.com, israelr@mellanox.com, idanb@mellanox.com, oren@mellanox.com, nitzanc@mellanox.com Subject: [PATCH 07/15] nvme-rdma: introduce nvme_rdma_sgl structure Date: Tue, 28 Apr 2020 16:11:27 +0300 Message-ID: <20200428131135.211521-8-maxg@mellanox.com> (raw) In-Reply-To: <20200428131135.211521-1-maxg@mellanox.com> From: Israel Rukshin <israelr@mellanox.com> Remove first_sgl pointer from struct nvme_rdma_request and use pointer arithmetic instead. The inline scatterlist, if exists, will be located right after the nvme_rdma_request. This patch is needed as a preparation for adding PI support. Signed-off-by: Israel Rukshin <israelr@mellanox.com> Reviewed-by: Max Gurtovoy <maxg@mellanox.com> Reviewed-by: Christoph Hellwig <hch@lst.de> --- drivers/nvme/host/rdma.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index cac8a93..4086874 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -48,6 +48,11 @@ struct nvme_rdma_qe { u64 dma; }; +struct nvme_rdma_sgl { + int nents; + struct sg_table sg_table; +}; + struct nvme_rdma_queue; struct nvme_rdma_request { struct nvme_request req; @@ -58,12 +63,10 @@ struct nvme_rdma_request { refcount_t ref; struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS]; u32 num_sge; - int nents; struct ib_reg_wr reg_wr; struct ib_cqe reg_cqe; struct nvme_rdma_queue *queue; - struct sg_table sg_table; - struct scatterlist first_sgl[]; + struct nvme_rdma_sgl data_sgl; }; enum nvme_rdma_queue_flags { @@ -1158,8 +1161,9 @@ static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue, req->mr = NULL; } - ib_dma_unmap_sg(ibdev, req->sg_table.sgl, req->nents, rq_dma_dir(rq)); - sg_free_table_chained(&req->sg_table, NVME_INLINE_SG_CNT); + ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, + rq_dma_dir(rq)); + sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); } static int nvme_rdma_set_sg_null(struct nvme_command *c) @@ -1178,7 +1182,7 @@ static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue, int count) { struct nvme_sgl_desc *sg = &c->common.dptr.sgl; - struct scatterlist *sgl = req->sg_table.sgl; + struct scatterlist *sgl = req->data_sgl.sg_table.sgl; struct ib_sge *sge = &req->sge[1]; u32 len = 0; int i; @@ -1203,8 +1207,8 @@ static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue, { struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl; - sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl)); - put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length); + sg->addr = cpu_to_le64(sg_dma_address(req->data_sgl.sg_table.sgl)); + put_unaligned_le24(sg_dma_len(req->data_sgl.sg_table.sgl), sg->length); put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key); sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4; return 0; @@ -1225,7 +1229,8 @@ static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue, * Align the MR to a 4K page size to match the ctrl page size and * the block virtual boundary. */ - nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, SZ_4K); + nr = ib_map_mr_sg(req->mr, req->data_sgl.sg_table.sgl, count, NULL, + SZ_4K); if (unlikely(nr < count)) { ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr); req->mr = NULL; @@ -1272,17 +1277,18 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue, if (!blk_rq_nr_phys_segments(rq)) return nvme_rdma_set_sg_null(c); - req->sg_table.sgl = req->first_sgl; - ret = sg_alloc_table_chained(&req->sg_table, - blk_rq_nr_phys_segments(rq), req->sg_table.sgl, + req->data_sgl.sg_table.sgl = (struct scatterlist *)(req + 1); + ret = sg_alloc_table_chained(&req->data_sgl.sg_table, + blk_rq_nr_phys_segments(rq), req->data_sgl.sg_table.sgl, NVME_INLINE_SG_CNT); if (ret) return -ENOMEM; - req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl); + req->data_sgl.nents = blk_rq_map_sg(rq->q, rq, + req->data_sgl.sg_table.sgl); - count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents, - rq_dma_dir(rq)); + count = ib_dma_map_sg(ibdev, req->data_sgl.sg_table.sgl, + req->data_sgl.nents, rq_dma_dir(rq)); if (unlikely(count <= 0)) { ret = -EIO; goto out_free_table; @@ -1311,9 +1317,10 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue, return 0; out_unmap_sg: - ib_dma_unmap_sg(ibdev, req->sg_table.sgl, req->nents, rq_dma_dir(rq)); + ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, + rq_dma_dir(rq)); out_free_table: - sg_free_table_chained(&req->sg_table, NVME_INLINE_SG_CNT); + sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); return ret; } -- 1.8.3.1 _______________________________________________ linux-nvme mailing list linux-nvme@lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-nvme
next prev parent reply index Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top 2020-04-28 13:11 [PATCH 00/15 V6] nvme-rdma/nvmet-rdma: Add metadata/T10-PI support Max Gurtovoy 2020-04-28 13:11 ` [PATCH 01/15] nvme: introduce namespace features flag Max Gurtovoy 2020-05-01 13:20 ` Christoph Hellwig 2020-05-01 14:24 ` Christoph Hellwig 2020-05-01 14:33 ` Max Gurtovoy 2020-04-28 13:11 ` [PATCH 02/15] nvme: introduce NVME_NS_METADATA_SUPPORTED flag Max Gurtovoy 2020-05-01 13:20 ` Christoph Hellwig 2020-04-28 13:11 ` [PATCH 03/15] nvme: make nvme_ns_has_pi accessible to transports Max Gurtovoy 2020-05-01 13:20 ` Christoph Hellwig 2020-04-28 13:11 ` [PATCH 04/15] nvme: enforce extended LBA format for fabrics metadata Max Gurtovoy 2020-05-01 13:21 ` Christoph Hellwig 2020-05-01 13:41 ` Christoph Hellwig 2020-04-28 13:11 ` [PATCH 05/15] nvme: introduce max_integrity_segments ctrl attribute Max Gurtovoy 2020-04-28 13:11 ` [PATCH 06/15] nvme: introduce NVME_INLINE_MD_SG_CNT Max Gurtovoy 2020-04-28 13:11 ` Max Gurtovoy [this message] 2020-04-28 13:11 ` [PATCH 08/15] nvme-rdma: add metadata/T10-PI support Max Gurtovoy 2020-05-01 14:26 ` Christoph Hellwig 2020-05-01 15:00 ` Max Gurtovoy 2020-04-28 13:11 ` [PATCH 09/15] nvmet: add metadata characteristics for a namespace Max Gurtovoy 2020-05-01 15:50 ` Christoph Hellwig 2020-04-28 13:11 ` [PATCH 10/15] nvmet: rename nvmet_rw_len to nvmet_rw_data_len Max Gurtovoy 2020-04-28 13:11 ` [PATCH 11/15] nvmet: rename nvmet_check_data_len to nvmet_check_transfer_len Max Gurtovoy 2020-04-28 13:11 ` [PATCH 12/15] nvme: add Metadata Capabilities enumerations Max Gurtovoy 2020-05-01 15:53 ` Christoph Hellwig 2020-05-03 12:43 ` Max Gurtovoy 2020-04-28 13:11 ` [PATCH 13/15] nvmet: add metadata/T10-PI support Max Gurtovoy 2020-05-01 15:58 ` Christoph Hellwig 2020-05-01 16:19 ` Christoph Hellwig 2020-04-28 13:11 ` [PATCH 14/15] nvmet: add metadata support for block devices Max Gurtovoy 2020-05-01 16:25 ` Christoph Hellwig 2020-04-28 13:11 ` [PATCH 15/15] nvmet-rdma: add metadata/T10-PI support Max Gurtovoy 2020-05-01 16:48 ` Christoph Hellwig 2020-05-03 16:29 ` Max Gurtovoy 2020-05-04 14:08 ` Christoph Hellwig 2020-05-04 14:19 ` Max Gurtovoy
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20200428131135.211521-8-maxg@mellanox.com \ --to=maxg@mellanox.com \ --cc=axboe@kernel.dk \ --cc=hch@lst.de \ --cc=idanb@mellanox.com \ --cc=israelr@mellanox.com \ --cc=jsmart2021@gmail.com \ --cc=kbusch@kernel.org \ --cc=linux-nvme@lists.infradead.org \ --cc=martin.petersen@oracle.com \ --cc=nitzanc@mellanox.com \ --cc=oren@mellanox.com \ --cc=sagi@grimberg.me \ --cc=shlomin@mellanox.com \ --cc=vladimirk@mellanox.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
Linux-NVME Archive on lore.kernel.org Archives are clonable: git clone --mirror https://lore.kernel.org/linux-nvme/0 linux-nvme/git/0.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 linux-nvme linux-nvme/ https://lore.kernel.org/linux-nvme \ linux-nvme@lists.infradead.org public-inbox-index linux-nvme Example config snippet for mirrors Newsgroup available over NNTP: nntp://nntp.lore.kernel.org/org.infradead.lists.linux-nvme AGPL code for this site: git clone https://public-inbox.org/public-inbox.git