From: Hannes Reinecke <hare@suse.de>
To: Shai Malin <smalin@marvell.com>,
netdev@vger.kernel.org, linux-nvme@lists.infradead.org,
sagi@grimberg.me, hch@lst.de, axboe@fb.com, kbusch@kernel.org
Cc: "David S . Miller davem @ davemloft . net --cc=Jakub Kicinski"
<kuba@kernel.org>,
aelior@marvell.com, mkalderon@marvell.com, okulkarni@marvell.com,
pkushwaha@marvell.com, malin1024@gmail.com,
Dean Balandin <dbalandin@marvell.com>
Subject: Re: [RFC PATCH v4 14/27] nvme-tcp-offload: Add IO level implementation
Date: Sat, 1 May 2021 18:38:54 +0200 [thread overview]
Message-ID: <bc7514a1-df4e-04b1-ecd2-ed4223bb4cd5@suse.de> (raw)
In-Reply-To: <20210429190926.5086-15-smalin@marvell.com>
On 4/29/21 9:09 PM, Shai Malin wrote:
> From: Dean Balandin <dbalandin@marvell.com>
>
> In this patch, we present the IO level functionality.
> The nvme-tcp-offload shall work on the IO-level, meaning the
> nvme-tcp-offload ULP module shall pass the request to the nvme-tcp-offload
> vendor driver and shall expect for the request compilation.
Request compilation? Not request completion?
> No additional handling is needed in between, this design will reduce the
> CPU utilization as we will describe below.
>
> The nvme-tcp-offload vendor driver shall register to nvme-tcp-offload ULP
> with the following IO-path ops:
> - init_req
> - send_req - in order to pass the request to the handling of the offload
> driver that shall pass it to the vendor specific device
> - poll_queue
>
> The vendor driver will manage the context from which the request will be
> executed and the request aggregations.
> Once the IO completed, the nvme-tcp-offload vendor driver shall call
> command.done() that shall invoke the nvme-tcp-offload ULP layer for
> completing the request.
>
> This patch also contains initial definition of nvme_tcp_ofld_queue_rq().
>
> Acked-by: Igor Russkikh <irusskikh@marvell.com>
> Signed-off-by: Dean Balandin <dbalandin@marvell.com>
> Signed-off-by: Prabhakar Kushwaha <pkushwaha@marvell.com>
> Signed-off-by: Omkar Kulkarni <okulkarni@marvell.com>
> Signed-off-by: Michal Kalderon <mkalderon@marvell.com>
> Signed-off-by: Ariel Elior <aelior@marvell.com>
> Signed-off-by: Shai Malin <smalin@marvell.com>
> ---
> drivers/nvme/host/tcp-offload.c | 95 ++++++++++++++++++++++++++++++---
> 1 file changed, 87 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/nvme/host/tcp-offload.c b/drivers/nvme/host/tcp-offload.c
> index 8ddce2257100..0cdf5a432208 100644
> --- a/drivers/nvme/host/tcp-offload.c
> +++ b/drivers/nvme/host/tcp-offload.c
> @@ -127,7 +127,10 @@ void nvme_tcp_ofld_req_done(struct nvme_tcp_ofld_req *req,
> union nvme_result *result,
> __le16 status)
> {
> - /* Placeholder - complete request with/without error */
> + struct request *rq = blk_mq_rq_from_pdu(req);
> +
> + if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), *result))
> + nvme_complete_rq(rq);
> }
>
> struct nvme_tcp_ofld_dev *
> @@ -686,6 +689,34 @@ static void nvme_tcp_ofld_free_ctrl(struct nvme_ctrl *nctrl)
> kfree(ctrl);
> }
>
> +static void nvme_tcp_ofld_set_sg_null(struct nvme_command *c)
> +{
> + struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
> +
> + sg->addr = 0;
> + sg->length = 0;
> + sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | NVME_SGL_FMT_TRANSPORT_A;
> +}
> +
> +inline void nvme_tcp_ofld_set_sg_inline(struct nvme_tcp_ofld_queue *queue,
> + struct nvme_command *c, u32 data_len)
> +{
> + struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
> +
> + sg->addr = cpu_to_le64(queue->ctrl->nctrl.icdoff);
> + sg->length = cpu_to_le32(data_len);
> + sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
> +}
> +
> +void nvme_tcp_ofld_map_data(struct nvme_command *c, u32 data_len)
> +{
> + struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
> +
> + sg->addr = 0;
> + sg->length = cpu_to_le32(data_len);
> + sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | NVME_SGL_FMT_TRANSPORT_A;
> +}
> +
> static void nvme_tcp_ofld_submit_async_event(struct nvme_ctrl *arg)
> {
> /* Placeholder - submit_async_event */
> @@ -841,9 +872,11 @@ nvme_tcp_ofld_init_request(struct blk_mq_tag_set *set,
> {
> struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(rq);
> struct nvme_tcp_ofld_ctrl *ctrl = set->driver_data;
> + int qid;
>
> - /* Placeholder - init request */
> -
> + qid = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
> + req->queue = &ctrl->queues[qid];
> + nvme_req(rq)->ctrl = &ctrl->nctrl;
> req->done = nvme_tcp_ofld_req_done;
> ctrl->dev->ops->init_req(req);
>
> @@ -858,16 +891,60 @@ EXPORT_SYMBOL_GPL(nvme_tcp_ofld_inline_data_size);
>
> static void nvme_tcp_ofld_commit_rqs(struct blk_mq_hw_ctx *hctx)
> {
> - /* Call ops->commit_rqs */
> + struct nvme_tcp_ofld_queue *queue = hctx->driver_data;
> + struct nvme_tcp_ofld_dev *dev = queue->dev;
> + struct nvme_tcp_ofld_ops *ops = dev->ops;
> +
> + ops->commit_rqs(queue);
> }
>
> static blk_status_t
> nvme_tcp_ofld_queue_rq(struct blk_mq_hw_ctx *hctx,
> const struct blk_mq_queue_data *bd)
> {
> - /* Call nvme_setup_cmd(...) */
> + struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(bd->rq);
> + struct nvme_tcp_ofld_queue *queue = hctx->driver_data;
> + struct nvme_tcp_ofld_ctrl *ctrl = queue->ctrl;
> + struct nvme_ns *ns = hctx->queue->queuedata;
> + struct nvme_tcp_ofld_dev *dev = queue->dev;
> + struct nvme_tcp_ofld_ops *ops = dev->ops;
> + struct nvme_command *nvme_cmd;
> + struct request *rq;
> + bool queue_ready;
> + u32 data_len;
> + int rc;
> +
> + queue_ready = test_bit(NVME_TCP_OFLD_Q_LIVE, &queue->flags);
> +
> + req->rq = bd->rq;
> + req->async = false;
> + rq = req->rq;
> +
> + if (!nvmf_check_ready(&ctrl->nctrl, req->rq, queue_ready))
> + return nvmf_fail_nonready_command(&ctrl->nctrl, req->rq);
> +
> + rc = nvme_setup_cmd(ns, req->rq, &req->nvme_cmd);
> + if (unlikely(rc))
> + return rc;
>
> - /* Call ops->send_req(...) */
> + blk_mq_start_request(req->rq);
> + req->last = bd->last;
> +
> + nvme_cmd = &req->nvme_cmd;
> + nvme_cmd->common.flags |= NVME_CMD_SGL_METABUF;
> +
> + data_len = blk_rq_nr_phys_segments(rq) ? blk_rq_payload_bytes(rq) : 0;
> + if (!data_len)
> + nvme_tcp_ofld_set_sg_null(&req->nvme_cmd);
> + else if ((rq_data_dir(rq) == WRITE) &&
> + data_len <= nvme_tcp_ofld_inline_data_size(queue))
> + nvme_tcp_ofld_set_sg_inline(queue, nvme_cmd, data_len);
> + else
> + nvme_tcp_ofld_map_data(nvme_cmd, data_len);
> +
> + rc = ops->send_req(req);
> + if (unlikely(rc))
> + return rc;
>
> return BLK_STS_OK;
> }
> @@ -940,9 +1017,11 @@ static int nvme_tcp_ofld_map_queues(struct blk_mq_tag_set *set)
>
> static int nvme_tcp_ofld_poll(struct blk_mq_hw_ctx *hctx)
> {
> - /* Placeholder - Implement polling mechanism */
> + struct nvme_tcp_ofld_queue *queue = hctx->driver_data;
> + struct nvme_tcp_ofld_dev *dev = queue->dev;
> + struct nvme_tcp_ofld_ops *ops = dev->ops;
>
> - return 0;
> + return ops->poll_queue(queue);
> }
>
> static struct blk_mq_ops nvme_tcp_ofld_mq_ops = {
>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Cheers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), Geschäftsführer: Felix Imendörffer
next prev parent reply other threads:[~2021-05-01 16:38 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-29 19:08 [RFC PATCH v4 00/27] NVMeTCP Offload ULP and QEDN Device Driver Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 01/27] qed: Add NVMeTCP Offload PF Level FW and HW HSI Shai Malin
2021-05-01 16:50 ` Hannes Reinecke
2021-05-03 15:23 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 02/27] qed: Add NVMeTCP Offload Connection " Shai Malin
2021-05-01 17:28 ` Hannes Reinecke
2021-05-03 15:25 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 03/27] qed: Add qed-NVMeTCP personality Shai Malin
2021-05-02 11:11 ` Hannes Reinecke
2021-05-03 15:26 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 04/27] qed: Add support of HW filter block Shai Malin
2021-05-02 11:13 ` Hannes Reinecke
2021-05-03 15:27 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 05/27] qed: Add NVMeTCP Offload IO Level FW and HW HSI Shai Malin
2021-05-02 11:22 ` Hannes Reinecke
2021-05-04 16:25 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 06/27] qed: Add NVMeTCP Offload IO Level FW Initializations Shai Malin
2021-05-02 11:24 ` Hannes Reinecke
2021-05-04 16:28 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 07/27] qed: Add IP services APIs support Shai Malin
2021-05-02 11:26 ` Hannes Reinecke
2021-05-03 15:44 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 08/27] nvme-tcp-offload: Add nvme-tcp-offload - NVMeTCP HW offload ULP Shai Malin
2021-05-01 12:18 ` Hannes Reinecke
2021-05-03 15:46 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 09/27] nvme-fabrics: Move NVMF_ALLOWED_OPTS and NVMF_REQUIRED_OPTS definitions Shai Malin
2021-05-01 12:19 ` Hannes Reinecke
2021-05-03 15:50 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 10/27] nvme-tcp-offload: Add device scan implementation Shai Malin
2021-05-01 12:25 ` Hannes Reinecke
2021-05-05 17:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 11/27] nvme-tcp-offload: Add controller level implementation Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 12/27] nvme-tcp-offload: Add controller level error recovery implementation Shai Malin
2021-05-01 16:29 ` Hannes Reinecke
2021-05-03 15:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 13/27] nvme-tcp-offload: Add queue level implementation Shai Malin
2021-05-01 16:36 ` Hannes Reinecke
2021-05-03 15:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 14/27] nvme-tcp-offload: Add IO " Shai Malin
2021-05-01 16:38 ` Hannes Reinecke [this message]
2021-05-04 16:34 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 15/27] nvme-tcp-offload: Add Timeout and ASYNC Support Shai Malin
2021-05-01 16:45 ` Hannes Reinecke
2021-05-04 16:49 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 16/27] qedn: Add qedn - Marvell's NVMeTCP HW offload vendor driver Shai Malin
2021-05-02 11:27 ` Hannes Reinecke
2021-05-04 16:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 17/27] qedn: Add qedn probe Shai Malin
2021-05-02 11:28 ` Hannes Reinecke
2021-05-08 11:01 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 18/27] qedn: Add qedn_claim_dev API support Shai Malin
2021-05-02 11:29 ` Hannes Reinecke
2021-05-07 13:57 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 19/27] qedn: Add IRQ and fast-path resources initializations Shai Malin
2021-05-02 11:32 ` Hannes Reinecke
2021-05-05 17:54 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 20/27] qedn: Add connection-level slowpath functionality Shai Malin
2021-05-02 11:37 ` Hannes Reinecke
2021-05-05 17:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 21/27] qedn: Add support of configuring HW filter block Shai Malin
2021-05-02 11:38 ` Hannes Reinecke
2021-05-05 17:57 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 22/27] qedn: Add IO level nvme_req and fw_cq workqueues Shai Malin
2021-05-02 11:42 ` Hannes Reinecke
2021-05-07 13:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 23/27] qedn: Add support of Task and SGL Shai Malin
2021-05-02 11:48 ` Hannes Reinecke
2021-05-07 14:00 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 24/27] qedn: Add support of NVME ICReq & ICResp Shai Malin
2021-05-02 11:53 ` Hannes Reinecke
2021-05-05 18:01 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 25/27] qedn: Add IO level fastpath functionality Shai Malin
2021-05-02 11:54 ` Hannes Reinecke
2021-05-05 18:04 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 26/27] qedn: Add Connection and IO level recovery flows Shai Malin
2021-05-02 11:57 ` Hannes Reinecke
2021-05-05 18:06 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 27/27] qedn: Add support of ASYNC Shai Malin
2021-05-02 11:59 ` Hannes Reinecke
2021-05-05 18:08 ` Shai Malin
2021-05-01 16:47 ` [RFC PATCH v4 00/27] NVMeTCP Offload ULP and QEDN Device Driver Hannes Reinecke
2021-05-03 15:13 ` Shai Malin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=bc7514a1-df4e-04b1-ecd2-ed4223bb4cd5@suse.de \
--to=hare@suse.de \
--cc=aelior@marvell.com \
--cc=axboe@fb.com \
--cc=dbalandin@marvell.com \
--cc=hch@lst.de \
--cc=kbusch@kernel.org \
--cc=kuba@kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=malin1024@gmail.com \
--cc=mkalderon@marvell.com \
--cc=netdev@vger.kernel.org \
--cc=okulkarni@marvell.com \
--cc=pkushwaha@marvell.com \
--cc=sagi@grimberg.me \
--cc=smalin@marvell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).