From: Hannes Reinecke <hare@suse.de>
To: Shai Malin <smalin@marvell.com>,
netdev@vger.kernel.org, linux-nvme@lists.infradead.org,
sagi@grimberg.me, hch@lst.de, axboe@fb.com, kbusch@kernel.org
Cc: "David S . Miller davem @ davemloft . net --cc=Jakub Kicinski"
<kuba@kernel.org>,
aelior@marvell.com, mkalderon@marvell.com, okulkarni@marvell.com,
pkushwaha@marvell.com, malin1024@gmail.com
Subject: Re: [RFC PATCH v4 15/27] nvme-tcp-offload: Add Timeout and ASYNC Support
Date: Sat, 1 May 2021 18:45:13 +0200 [thread overview]
Message-ID: <d762b4f0-c048-f9bd-a58b-fdbf3804e6a7@suse.de> (raw)
In-Reply-To: <20210429190926.5086-16-smalin@marvell.com>
On 4/29/21 9:09 PM, Shai Malin wrote:
> In this patch, we present the nvme-tcp-offload timeout support
> nvme_tcp_ofld_timeout() and ASYNC support
> nvme_tcp_ofld_submit_async_event().
>
> Acked-by: Igor Russkikh <irusskikh@marvell.com>
> Signed-off-by: Prabhakar Kushwaha <pkushwaha@marvell.com>
> Signed-off-by: Omkar Kulkarni <okulkarni@marvell.com>
> Signed-off-by: Michal Kalderon <mkalderon@marvell.com>
> Signed-off-by: Ariel Elior <aelior@marvell.com>
> Signed-off-by: Shai Malin <smalin@marvell.com>
> ---
> drivers/nvme/host/tcp-offload.c | 85 ++++++++++++++++++++++++++++++++-
> drivers/nvme/host/tcp-offload.h | 2 +
> 2 files changed, 86 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/nvme/host/tcp-offload.c b/drivers/nvme/host/tcp-offload.c
> index 0cdf5a432208..1d62f921f109 100644
> --- a/drivers/nvme/host/tcp-offload.c
> +++ b/drivers/nvme/host/tcp-offload.c
> @@ -133,6 +133,26 @@ void nvme_tcp_ofld_req_done(struct nvme_tcp_ofld_req *req,
> nvme_complete_rq(rq);
> }
>
> +/**
> + * nvme_tcp_ofld_async_req_done() - NVMeTCP Offload request done callback
> + * function for async request. Pointed to by nvme_tcp_ofld_req->done.
> + * Handles both NVME_TCP_F_DATA_SUCCESS flag and NVMe CQ.
> + * @req: NVMeTCP offload request to complete.
> + * @result: The nvme_result.
> + * @status: The completion status.
> + *
> + * API function that allows the vendor specific offload driver to report request
> + * completions to the common offload layer.
> + */
> +void nvme_tcp_ofld_async_req_done(struct nvme_tcp_ofld_req *req,
> + union nvme_result *result, __le16 status)
> +{
> + struct nvme_tcp_ofld_queue *queue = req->queue;
> + struct nvme_tcp_ofld_ctrl *ctrl = queue->ctrl;
> +
> + nvme_complete_async_event(&ctrl->nctrl, status, result);
> +}
> +
> struct nvme_tcp_ofld_dev *
> nvme_tcp_ofld_lookup_dev(struct nvme_tcp_ofld_ctrl *ctrl)
> {
> @@ -719,7 +739,23 @@ void nvme_tcp_ofld_map_data(struct nvme_command *c, u32 data_len)
>
> static void nvme_tcp_ofld_submit_async_event(struct nvme_ctrl *arg)
> {
> - /* Placeholder - submit_async_event */
> + struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(arg);
> + struct nvme_tcp_ofld_queue *queue = &ctrl->queues[0];
> + struct nvme_tcp_ofld_dev *dev = queue->dev;
> + struct nvme_tcp_ofld_ops *ops = dev->ops;
> +
> + ctrl->async_req.nvme_cmd.common.opcode = nvme_admin_async_event;
> + ctrl->async_req.nvme_cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
> + ctrl->async_req.nvme_cmd.common.flags |= NVME_CMD_SGL_METABUF;
> +
> + nvme_tcp_ofld_set_sg_null(&ctrl->async_req.nvme_cmd);
> +
> + ctrl->async_req.async = true;
> + ctrl->async_req.queue = queue;
> + ctrl->async_req.last = true;
> + ctrl->async_req.done = nvme_tcp_ofld_async_req_done;
> +
> + ops->send_req(&ctrl->async_req);
> }
>
> static void
> @@ -1024,6 +1060,51 @@ static int nvme_tcp_ofld_poll(struct blk_mq_hw_ctx *hctx)
> return ops->poll_queue(queue);
> }
>
> +static void nvme_tcp_ofld_complete_timed_out(struct request *rq)
> +{
> + struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(rq);
> + struct nvme_ctrl *nctrl = &req->queue->ctrl->nctrl;
> +
> + nvme_tcp_ofld_stop_queue(nctrl, nvme_tcp_ofld_qid(req->queue));
> + if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {
> + nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
> + blk_mq_complete_request(rq);
> + }
> +}
> +
> +static enum blk_eh_timer_return nvme_tcp_ofld_timeout(struct request *rq, bool reserved)
> +{
> + struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(rq);
> + struct nvme_tcp_ofld_ctrl *ctrl = req->queue->ctrl;
> +
> + dev_warn(ctrl->nctrl.device,
> + "queue %d: timeout request %#x type %d\n",
> + nvme_tcp_ofld_qid(req->queue), rq->tag, req->nvme_cmd.common.opcode);
> +
> + if (ctrl->nctrl.state != NVME_CTRL_LIVE) {
> + /*
> + * If we are resetting, connecting or deleting we should
> + * complete immediately because we may block controller
> + * teardown or setup sequence
> + * - ctrl disable/shutdown fabrics requests
> + * - connect requests
> + * - initialization admin requests
> + * - I/O requests that entered after unquiescing and
> + * the controller stopped responding
> + *
> + * All other requests should be cancelled by the error
> + * recovery work, so it's fine that we fail it here.
> + */
> + nvme_tcp_ofld_complete_timed_out(rq);
> +
> + return BLK_EH_DONE;
> + }
And this particular error code has been causing _so_ _many_ issues
during testing, that I'd rather get rid of it altogether.
But probably not your fault, your just copying what tcp and rdma is doing.
> +
> + nvme_tcp_ofld_error_recovery(&ctrl->nctrl);
> +
> + return BLK_EH_RESET_TIMER;
> +}
> +
> static struct blk_mq_ops nvme_tcp_ofld_mq_ops = {
> .queue_rq = nvme_tcp_ofld_queue_rq,
> .commit_rqs = nvme_tcp_ofld_commit_rqs,
> @@ -1031,6 +1112,7 @@ static struct blk_mq_ops nvme_tcp_ofld_mq_ops = {
> .init_request = nvme_tcp_ofld_init_request,
> .exit_request = nvme_tcp_ofld_exit_request,
> .init_hctx = nvme_tcp_ofld_init_hctx,
> + .timeout = nvme_tcp_ofld_timeout,
> .map_queues = nvme_tcp_ofld_map_queues,
> .poll = nvme_tcp_ofld_poll,
> };
> @@ -1041,6 +1123,7 @@ static struct blk_mq_ops nvme_tcp_ofld_admin_mq_ops = {
> .init_request = nvme_tcp_ofld_init_request,
> .exit_request = nvme_tcp_ofld_exit_request,
> .init_hctx = nvme_tcp_ofld_init_admin_hctx,
> + .timeout = nvme_tcp_ofld_timeout,
> };
>
> static const struct nvme_ctrl_ops nvme_tcp_ofld_ctrl_ops = {
> diff --git a/drivers/nvme/host/tcp-offload.h b/drivers/nvme/host/tcp-offload.h
> index d82645fcf9da..275a7e2d9d8a 100644
> --- a/drivers/nvme/host/tcp-offload.h
> +++ b/drivers/nvme/host/tcp-offload.h
> @@ -110,6 +110,8 @@ struct nvme_tcp_ofld_ctrl {
> /* Connectivity params */
> struct nvme_tcp_ofld_ctrl_con_params conn_params;
>
> + struct nvme_tcp_ofld_req async_req;
> +
> /* Vendor specific driver context */
> void *private_data;
> };
>
So:
Reviewed-by: Hannes Reinecke <hare@suse.de>
Cheers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), Geschäftsführer: Felix Imendörffer
next prev parent reply other threads:[~2021-05-01 16:45 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-29 19:08 [RFC PATCH v4 00/27] NVMeTCP Offload ULP and QEDN Device Driver Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 01/27] qed: Add NVMeTCP Offload PF Level FW and HW HSI Shai Malin
2021-05-01 16:50 ` Hannes Reinecke
2021-05-03 15:23 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 02/27] qed: Add NVMeTCP Offload Connection " Shai Malin
2021-05-01 17:28 ` Hannes Reinecke
2021-05-03 15:25 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 03/27] qed: Add qed-NVMeTCP personality Shai Malin
2021-05-02 11:11 ` Hannes Reinecke
2021-05-03 15:26 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 04/27] qed: Add support of HW filter block Shai Malin
2021-05-02 11:13 ` Hannes Reinecke
2021-05-03 15:27 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 05/27] qed: Add NVMeTCP Offload IO Level FW and HW HSI Shai Malin
2021-05-02 11:22 ` Hannes Reinecke
2021-05-04 16:25 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 06/27] qed: Add NVMeTCP Offload IO Level FW Initializations Shai Malin
2021-05-02 11:24 ` Hannes Reinecke
2021-05-04 16:28 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 07/27] qed: Add IP services APIs support Shai Malin
2021-05-02 11:26 ` Hannes Reinecke
2021-05-03 15:44 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 08/27] nvme-tcp-offload: Add nvme-tcp-offload - NVMeTCP HW offload ULP Shai Malin
2021-05-01 12:18 ` Hannes Reinecke
2021-05-03 15:46 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 09/27] nvme-fabrics: Move NVMF_ALLOWED_OPTS and NVMF_REQUIRED_OPTS definitions Shai Malin
2021-05-01 12:19 ` Hannes Reinecke
2021-05-03 15:50 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 10/27] nvme-tcp-offload: Add device scan implementation Shai Malin
2021-05-01 12:25 ` Hannes Reinecke
2021-05-05 17:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 11/27] nvme-tcp-offload: Add controller level implementation Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 12/27] nvme-tcp-offload: Add controller level error recovery implementation Shai Malin
2021-05-01 16:29 ` Hannes Reinecke
2021-05-03 15:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 13/27] nvme-tcp-offload: Add queue level implementation Shai Malin
2021-05-01 16:36 ` Hannes Reinecke
2021-05-03 15:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 14/27] nvme-tcp-offload: Add IO " Shai Malin
2021-05-01 16:38 ` Hannes Reinecke
2021-05-04 16:34 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 15/27] nvme-tcp-offload: Add Timeout and ASYNC Support Shai Malin
2021-05-01 16:45 ` Hannes Reinecke [this message]
2021-05-04 16:49 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 16/27] qedn: Add qedn - Marvell's NVMeTCP HW offload vendor driver Shai Malin
2021-05-02 11:27 ` Hannes Reinecke
2021-05-04 16:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 17/27] qedn: Add qedn probe Shai Malin
2021-05-02 11:28 ` Hannes Reinecke
2021-05-08 11:01 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 18/27] qedn: Add qedn_claim_dev API support Shai Malin
2021-05-02 11:29 ` Hannes Reinecke
2021-05-07 13:57 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 19/27] qedn: Add IRQ and fast-path resources initializations Shai Malin
2021-05-02 11:32 ` Hannes Reinecke
2021-05-05 17:54 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 20/27] qedn: Add connection-level slowpath functionality Shai Malin
2021-05-02 11:37 ` Hannes Reinecke
2021-05-05 17:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 21/27] qedn: Add support of configuring HW filter block Shai Malin
2021-05-02 11:38 ` Hannes Reinecke
2021-05-05 17:57 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 22/27] qedn: Add IO level nvme_req and fw_cq workqueues Shai Malin
2021-05-02 11:42 ` Hannes Reinecke
2021-05-07 13:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 23/27] qedn: Add support of Task and SGL Shai Malin
2021-05-02 11:48 ` Hannes Reinecke
2021-05-07 14:00 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 24/27] qedn: Add support of NVME ICReq & ICResp Shai Malin
2021-05-02 11:53 ` Hannes Reinecke
2021-05-05 18:01 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 25/27] qedn: Add IO level fastpath functionality Shai Malin
2021-05-02 11:54 ` Hannes Reinecke
2021-05-05 18:04 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 26/27] qedn: Add Connection and IO level recovery flows Shai Malin
2021-05-02 11:57 ` Hannes Reinecke
2021-05-05 18:06 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 27/27] qedn: Add support of ASYNC Shai Malin
2021-05-02 11:59 ` Hannes Reinecke
2021-05-05 18:08 ` Shai Malin
2021-05-01 16:47 ` [RFC PATCH v4 00/27] NVMeTCP Offload ULP and QEDN Device Driver Hannes Reinecke
2021-05-03 15:13 ` Shai Malin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=d762b4f0-c048-f9bd-a58b-fdbf3804e6a7@suse.de \
--to=hare@suse.de \
--cc=aelior@marvell.com \
--cc=axboe@fb.com \
--cc=hch@lst.de \
--cc=kbusch@kernel.org \
--cc=kuba@kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=malin1024@gmail.com \
--cc=mkalderon@marvell.com \
--cc=netdev@vger.kernel.org \
--cc=okulkarni@marvell.com \
--cc=pkushwaha@marvell.com \
--cc=sagi@grimberg.me \
--cc=smalin@marvell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).