From: Shai Malin <smalin@marvell.com>
To: <netdev@vger.kernel.org>, <linux-nvme@lists.infradead.org>,
<sagi@grimberg.me>, <hch@lst.de>, <axboe@fb.com>,
<kbusch@kernel.org>
Cc: "David S . Miller davem @ davemloft . net --cc=Jakub Kicinski"
<kuba@kernel.org>,
smalin@marvell.com, aelior@marvell.com, mkalderon@marvell.com,
okulkarni@marvell.com, pkushwaha@marvell.com,
malin1024@gmail.com, "Arie Gershberg" <agershberg@marvell.com>
Subject: [RFC PATCH v4 12/27] nvme-tcp-offload: Add controller level error recovery implementation
Date: Thu, 29 Apr 2021 22:09:11 +0300 [thread overview]
Message-ID: <20210429190926.5086-13-smalin@marvell.com> (raw)
In-Reply-To: <20210429190926.5086-1-smalin@marvell.com>
From: Arie Gershberg <agershberg@marvell.com>
In this patch, we implement controller level error handling and recovery.
Upon an error discovered by the ULP or reset controller initiated by the
nvme-core (using reset_ctrl workqueue), the ULP will initiate a controller
recovery which includes teardown and re-connect of all queues.
Acked-by: Igor Russkikh <irusskikh@marvell.com>
Signed-off-by: Arie Gershberg <agershberg@marvell.com>
Signed-off-by: Prabhakar Kushwaha <pkushwaha@marvell.com>
Signed-off-by: Omkar Kulkarni <okulkarni@marvell.com>
Signed-off-by: Michal Kalderon <mkalderon@marvell.com>
Signed-off-by: Ariel Elior <aelior@marvell.com>
Signed-off-by: Shai Malin <smalin@marvell.com>
---
drivers/nvme/host/tcp-offload.c | 138 +++++++++++++++++++++++++++++++-
drivers/nvme/host/tcp-offload.h | 1 +
2 files changed, 137 insertions(+), 2 deletions(-)
diff --git a/drivers/nvme/host/tcp-offload.c b/drivers/nvme/host/tcp-offload.c
index 59e1955e02ec..9082b11c133f 100644
--- a/drivers/nvme/host/tcp-offload.c
+++ b/drivers/nvme/host/tcp-offload.c
@@ -74,6 +74,23 @@ void nvme_tcp_ofld_unregister_dev(struct nvme_tcp_ofld_dev *dev)
}
EXPORT_SYMBOL_GPL(nvme_tcp_ofld_unregister_dev);
+/**
+ * nvme_tcp_ofld_error_recovery() - NVMeTCP Offload Library error recovery.
+ * function.
+ * @nctrl: NVMe controller instance to change to resetting.
+ *
+ * API function that change the controller state to resseting.
+ * Part of the overall controller reset sequence.
+ */
+void nvme_tcp_ofld_error_recovery(struct nvme_ctrl *nctrl)
+{
+ if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_RESETTING))
+ return;
+
+ queue_work(nvme_reset_wq, &to_tcp_ofld_ctrl(nctrl)->err_work);
+}
+EXPORT_SYMBOL_GPL(nvme_tcp_ofld_error_recovery);
+
/**
* nvme_tcp_ofld_report_queue_err() - NVMeTCP Offload report error event
* callback function. Pointed to by nvme_tcp_ofld_queue->report_err.
@@ -84,7 +101,8 @@ EXPORT_SYMBOL_GPL(nvme_tcp_ofld_unregister_dev);
*/
int nvme_tcp_ofld_report_queue_err(struct nvme_tcp_ofld_queue *queue)
{
- /* Placeholder - invoke error recovery flow */
+ pr_err("nvme-tcp-offload queue error\n");
+ nvme_tcp_ofld_error_recovery(&queue->ctrl->nctrl);
return 0;
}
@@ -296,6 +314,28 @@ nvme_tcp_ofld_configure_io_queues(struct nvme_ctrl *nctrl, bool new)
return rc;
}
+static void nvme_tcp_ofld_reconnect_or_remove(struct nvme_ctrl *nctrl)
+{
+ /* If we are resetting/deleting then do nothing */
+ if (nctrl->state != NVME_CTRL_CONNECTING) {
+ WARN_ON_ONCE(nctrl->state == NVME_CTRL_NEW ||
+ nctrl->state == NVME_CTRL_LIVE);
+
+ return;
+ }
+
+ if (nvmf_should_reconnect(nctrl)) {
+ dev_info(nctrl->device, "Reconnecting in %d seconds...\n",
+ nctrl->opts->reconnect_delay);
+ queue_delayed_work(nvme_wq,
+ &to_tcp_ofld_ctrl(nctrl)->connect_work,
+ nctrl->opts->reconnect_delay * HZ);
+ } else {
+ dev_info(nctrl->device, "Removing controller...\n");
+ nvme_delete_ctrl(nctrl);
+ }
+}
+
static int nvme_tcp_ofld_setup_ctrl(struct nvme_ctrl *nctrl, bool new)
{
struct nvmf_ctrl_options *opts = nctrl->opts;
@@ -407,10 +447,68 @@ nvme_tcp_ofld_teardown_io_queues(struct nvme_ctrl *nctrl, bool remove)
/* Placeholder - teardown_io_queues */
}
+static void nvme_tcp_ofld_reconnect_ctrl_work(struct work_struct *work)
+{
+ struct nvme_tcp_ofld_ctrl *ctrl =
+ container_of(to_delayed_work(work),
+ struct nvme_tcp_ofld_ctrl,
+ connect_work);
+ struct nvme_ctrl *nctrl = &ctrl->nctrl;
+
+ ++nctrl->nr_reconnects;
+
+ if (ctrl->dev->ops->setup_ctrl(ctrl, false))
+ goto requeue;
+
+ if (nvme_tcp_ofld_setup_ctrl(nctrl, false))
+ goto release_and_requeue;
+
+ dev_info(nctrl->device, "Successfully reconnected (%d attempt)\n",
+ nctrl->nr_reconnects);
+
+ nctrl->nr_reconnects = 0;
+
+ return;
+
+release_and_requeue:
+ ctrl->dev->ops->release_ctrl(ctrl);
+requeue:
+ dev_info(nctrl->device, "Failed reconnect attempt %d\n",
+ nctrl->nr_reconnects);
+ nvme_tcp_ofld_reconnect_or_remove(nctrl);
+}
+
+static void nvme_tcp_ofld_error_recovery_work(struct work_struct *work)
+{
+ struct nvme_tcp_ofld_ctrl *ctrl =
+ container_of(work, struct nvme_tcp_ofld_ctrl, err_work);
+ struct nvme_ctrl *nctrl = &ctrl->nctrl;
+
+ nvme_stop_keep_alive(nctrl);
+ nvme_tcp_ofld_teardown_io_queues(nctrl, false);
+ /* unquiesce to fail fast pending requests */
+ nvme_start_queues(nctrl);
+ nvme_tcp_ofld_teardown_admin_queue(nctrl, false);
+ blk_mq_unquiesce_queue(nctrl->admin_q);
+
+ if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_CONNECTING)) {
+ /* state change failure is ok if we started nctrl delete */
+ WARN_ON_ONCE(nctrl->state != NVME_CTRL_DELETING &&
+ nctrl->state != NVME_CTRL_DELETING_NOIO);
+
+ return;
+ }
+
+ nvme_tcp_ofld_reconnect_or_remove(nctrl);
+}
+
static void
nvme_tcp_ofld_teardown_ctrl(struct nvme_ctrl *nctrl, bool shutdown)
{
- /* Placeholder - err_work and connect_work */
+ struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
+
+ cancel_work_sync(&ctrl->err_work);
+ cancel_delayed_work_sync(&ctrl->connect_work);
nvme_tcp_ofld_teardown_io_queues(nctrl, shutdown);
blk_mq_quiesce_queue(nctrl->admin_q);
if (shutdown)
@@ -425,6 +523,38 @@ static void nvme_tcp_ofld_delete_ctrl(struct nvme_ctrl *nctrl)
nvme_tcp_ofld_teardown_ctrl(nctrl, true);
}
+static void nvme_tcp_ofld_reset_ctrl_work(struct work_struct *work)
+{
+ struct nvme_ctrl *nctrl =
+ container_of(work, struct nvme_ctrl, reset_work);
+ struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
+
+ nvme_stop_ctrl(nctrl);
+ nvme_tcp_ofld_teardown_ctrl(nctrl, false);
+
+ if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_CONNECTING)) {
+ /* state change failure is ok if we started ctrl delete */
+ WARN_ON_ONCE(nctrl->state != NVME_CTRL_DELETING &&
+ nctrl->state != NVME_CTRL_DELETING_NOIO);
+
+ return;
+ }
+
+ if (ctrl->dev->ops->setup_ctrl(ctrl, false))
+ goto out_fail;
+
+ if (nvme_tcp_ofld_setup_ctrl(nctrl, false))
+ goto release_ctrl;
+
+ return;
+
+release_ctrl:
+ ctrl->dev->ops->release_ctrl(ctrl);
+out_fail:
+ ++nctrl->nr_reconnects;
+ nvme_tcp_ofld_reconnect_or_remove(nctrl);
+}
+
static int
nvme_tcp_ofld_init_request(struct blk_mq_tag_set *set,
struct request *rq,
@@ -521,6 +651,10 @@ nvme_tcp_ofld_create_ctrl(struct device *ndev, struct nvmf_ctrl_options *opts)
opts->nr_poll_queues + 1;
nctrl->sqsize = opts->queue_size - 1;
nctrl->kato = opts->kato;
+ INIT_DELAYED_WORK(&ctrl->connect_work,
+ nvme_tcp_ofld_reconnect_ctrl_work);
+ INIT_WORK(&ctrl->err_work, nvme_tcp_ofld_error_recovery_work);
+ INIT_WORK(&nctrl->reset_work, nvme_tcp_ofld_reset_ctrl_work);
if (!(opts->mask & NVMF_OPT_TRSVCID)) {
opts->trsvcid =
kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
diff --git a/drivers/nvme/host/tcp-offload.h b/drivers/nvme/host/tcp-offload.h
index 9fd270240eaa..b23b1d7ea6fa 100644
--- a/drivers/nvme/host/tcp-offload.h
+++ b/drivers/nvme/host/tcp-offload.h
@@ -204,3 +204,4 @@ struct nvme_tcp_ofld_ops {
/* Exported functions for lower vendor specific offload drivers */
int nvme_tcp_ofld_register_dev(struct nvme_tcp_ofld_dev *dev);
void nvme_tcp_ofld_unregister_dev(struct nvme_tcp_ofld_dev *dev);
+void nvme_tcp_ofld_error_recovery(struct nvme_ctrl *nctrl);
--
2.22.0
next prev parent reply other threads:[~2021-04-29 19:18 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-29 19:08 [RFC PATCH v4 00/27] NVMeTCP Offload ULP and QEDN Device Driver Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 01/27] qed: Add NVMeTCP Offload PF Level FW and HW HSI Shai Malin
2021-05-01 16:50 ` Hannes Reinecke
2021-05-03 15:23 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 02/27] qed: Add NVMeTCP Offload Connection " Shai Malin
2021-05-01 17:28 ` Hannes Reinecke
2021-05-03 15:25 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 03/27] qed: Add qed-NVMeTCP personality Shai Malin
2021-05-02 11:11 ` Hannes Reinecke
2021-05-03 15:26 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 04/27] qed: Add support of HW filter block Shai Malin
2021-05-02 11:13 ` Hannes Reinecke
2021-05-03 15:27 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 05/27] qed: Add NVMeTCP Offload IO Level FW and HW HSI Shai Malin
2021-05-02 11:22 ` Hannes Reinecke
2021-05-04 16:25 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 06/27] qed: Add NVMeTCP Offload IO Level FW Initializations Shai Malin
2021-05-02 11:24 ` Hannes Reinecke
2021-05-04 16:28 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 07/27] qed: Add IP services APIs support Shai Malin
2021-05-02 11:26 ` Hannes Reinecke
2021-05-03 15:44 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 08/27] nvme-tcp-offload: Add nvme-tcp-offload - NVMeTCP HW offload ULP Shai Malin
2021-05-01 12:18 ` Hannes Reinecke
2021-05-03 15:46 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 09/27] nvme-fabrics: Move NVMF_ALLOWED_OPTS and NVMF_REQUIRED_OPTS definitions Shai Malin
2021-05-01 12:19 ` Hannes Reinecke
2021-05-03 15:50 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 10/27] nvme-tcp-offload: Add device scan implementation Shai Malin
2021-05-01 12:25 ` Hannes Reinecke
2021-05-05 17:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 11/27] nvme-tcp-offload: Add controller level implementation Shai Malin
2021-04-29 19:09 ` Shai Malin [this message]
2021-05-01 16:29 ` [RFC PATCH v4 12/27] nvme-tcp-offload: Add controller level error recovery implementation Hannes Reinecke
2021-05-03 15:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 13/27] nvme-tcp-offload: Add queue level implementation Shai Malin
2021-05-01 16:36 ` Hannes Reinecke
2021-05-03 15:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 14/27] nvme-tcp-offload: Add IO " Shai Malin
2021-05-01 16:38 ` Hannes Reinecke
2021-05-04 16:34 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 15/27] nvme-tcp-offload: Add Timeout and ASYNC Support Shai Malin
2021-05-01 16:45 ` Hannes Reinecke
2021-05-04 16:49 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 16/27] qedn: Add qedn - Marvell's NVMeTCP HW offload vendor driver Shai Malin
2021-05-02 11:27 ` Hannes Reinecke
2021-05-04 16:52 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 17/27] qedn: Add qedn probe Shai Malin
2021-05-02 11:28 ` Hannes Reinecke
2021-05-08 11:01 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 18/27] qedn: Add qedn_claim_dev API support Shai Malin
2021-05-02 11:29 ` Hannes Reinecke
2021-05-07 13:57 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 19/27] qedn: Add IRQ and fast-path resources initializations Shai Malin
2021-05-02 11:32 ` Hannes Reinecke
2021-05-05 17:54 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 20/27] qedn: Add connection-level slowpath functionality Shai Malin
2021-05-02 11:37 ` Hannes Reinecke
2021-05-05 17:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 21/27] qedn: Add support of configuring HW filter block Shai Malin
2021-05-02 11:38 ` Hannes Reinecke
2021-05-05 17:57 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 22/27] qedn: Add IO level nvme_req and fw_cq workqueues Shai Malin
2021-05-02 11:42 ` Hannes Reinecke
2021-05-07 13:56 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 23/27] qedn: Add support of Task and SGL Shai Malin
2021-05-02 11:48 ` Hannes Reinecke
2021-05-07 14:00 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 24/27] qedn: Add support of NVME ICReq & ICResp Shai Malin
2021-05-02 11:53 ` Hannes Reinecke
2021-05-05 18:01 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 25/27] qedn: Add IO level fastpath functionality Shai Malin
2021-05-02 11:54 ` Hannes Reinecke
2021-05-05 18:04 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 26/27] qedn: Add Connection and IO level recovery flows Shai Malin
2021-05-02 11:57 ` Hannes Reinecke
2021-05-05 18:06 ` Shai Malin
2021-04-29 19:09 ` [RFC PATCH v4 27/27] qedn: Add support of ASYNC Shai Malin
2021-05-02 11:59 ` Hannes Reinecke
2021-05-05 18:08 ` Shai Malin
2021-05-01 16:47 ` [RFC PATCH v4 00/27] NVMeTCP Offload ULP and QEDN Device Driver Hannes Reinecke
2021-05-03 15:13 ` Shai Malin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210429190926.5086-13-smalin@marvell.com \
--to=smalin@marvell.com \
--cc=aelior@marvell.com \
--cc=agershberg@marvell.com \
--cc=axboe@fb.com \
--cc=hch@lst.de \
--cc=kbusch@kernel.org \
--cc=kuba@kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=malin1024@gmail.com \
--cc=mkalderon@marvell.com \
--cc=netdev@vger.kernel.org \
--cc=okulkarni@marvell.com \
--cc=pkushwaha@marvell.com \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).