From: Sreekanth Reddy <sreekanth.reddy@broadcom.com>
To: linux-scsi@vger.kernel.org
Cc: martin.petersen@oracle.com, mpi3mr-linuxdrv.pdl@broadcom.com,
Sreekanth Reddy <sreekanth.reddy@broadcom.com>
Subject: [PATCH 11/25] mpi3mr: Fault IOC when internal commands gets timeout
Date: Mon, 20 Dec 2021 19:41:45 +0530 [thread overview]
Message-ID: <20211220141159.16117-12-sreekanth.reddy@broadcom.com> (raw)
In-Reply-To: <20211220141159.16117-1-sreekanth.reddy@broadcom.com>
[-- Attachment #1: Type: text/plain, Size: 8129 bytes --]
Save snapdump and fault the controller with the given
reason code if it is already not in the fault or not in
asynchronous reset. So that soft reset is issued
from the watchdog thread. This will also be used
to handle initialization time faults/resets/timeout
as in those cases immediate soft reset invocation is
not required.
Signed-off-by: Sreekanth Reddy <sreekanth.reddy@broadcom.com>
---
drivers/scsi/mpi3mr/mpi3mr.h | 1 +
drivers/scsi/mpi3mr/mpi3mr_fw.c | 114 ++++++++++++++++++--------------
2 files changed, 67 insertions(+), 48 deletions(-)
diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h
index fdbedf2..55a07f9 100644
--- a/drivers/scsi/mpi3mr/mpi3mr.h
+++ b/drivers/scsi/mpi3mr/mpi3mr.h
@@ -916,5 +916,6 @@ void mpi3mr_flush_host_io(struct mpi3mr_ioc *mrioc);
void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc);
void mpi3mr_rfresh_tgtdevs(struct mpi3mr_ioc *mrioc);
void mpi3mr_flush_delayed_rmhs_list(struct mpi3mr_ioc *mrioc);
+void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code);
#endif /*MPI3MR_H_INCLUDED*/
diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
index 6b534ed..b6d4e9d 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_fw.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
@@ -1391,13 +1391,9 @@ static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
wait_for_completion_timeout(&mrioc->init_cmds.done,
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- ioc_err(mrioc, "Issue DelRepQ: command timed out\n");
- mpi3mr_set_diagsave(mrioc);
- mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
+ ioc_err(mrioc, "delete reply queue timed out\n");
+ mpi3mr_check_rh_fault_ioc(mrioc,
MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
- mrioc->unrecoverable = 1;
-
retval = -1;
goto out_unlock;
}
@@ -1617,12 +1613,9 @@ static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
wait_for_completion_timeout(&mrioc->init_cmds.done,
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- ioc_err(mrioc, "CreateRepQ: command timed out\n");
- mpi3mr_set_diagsave(mrioc);
- mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
+ ioc_err(mrioc, "create reply queue timed out\n");
+ mpi3mr_check_rh_fault_ioc(mrioc,
MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
- mrioc->unrecoverable = 1;
retval = -1;
goto out_unlock;
}
@@ -1724,12 +1717,9 @@ static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
wait_for_completion_timeout(&mrioc->init_cmds.done,
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- ioc_err(mrioc, "CreateReqQ: command timed out\n");
- mpi3mr_set_diagsave(mrioc);
- if (mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
- MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT))
- mrioc->unrecoverable = 1;
+ ioc_err(mrioc, "create request queue timed out\n");
+ mpi3mr_check_rh_fault_ioc(mrioc,
+ MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT);
retval = -1;
goto out_unlock;
}
@@ -1902,6 +1892,42 @@ out:
return retval;
}
+/**
+ * mpi3mr_check_rh_fault_ioc - check reset history and fault
+ * controller
+ * @mrioc: Adapter instance reference
+ * @reason_code, reason code for the fault.
+ *
+ * This routine will save snapdump and fault the controller with
+ * the given reason code if it is not already in the fault or
+ * not asynchronosuly reset. This will be used to handle
+ * initilaization time faults/resets/timeout as in those cases
+ * immediate soft reset invocation is not required.
+ *
+ * Return: None.
+ */
+void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
+{
+ u32 ioc_status, host_diagnostic, timeout;
+
+ ioc_status = readl(&mrioc->sysif_regs->ioc_status);
+ if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
+ (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
+ mpi3mr_print_fault_info(mrioc);
+ return;
+ }
+ mpi3mr_set_diagsave(mrioc);
+ mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
+ reason_code);
+ timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
+ do {
+ host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
+ if (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
+ break;
+ msleep(100);
+ } while (--timeout);
+}
+
/**
* mpi3mr_sync_timestamp - Issue time stamp sync request
* @mrioc: Adapter reference
@@ -2025,6 +2051,8 @@ static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
ioc_err(mrioc, "get package version timed out\n");
+ mpi3mr_check_rh_fault_ioc(mrioc,
+ MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT);
retval = -1;
goto out_unlock;
}
@@ -2344,12 +2372,9 @@ static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
wait_for_completion_timeout(&mrioc->init_cmds.done,
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- ioc_err(mrioc, "Issue IOCFacts: command timed out\n");
- mpi3mr_set_diagsave(mrioc);
- mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
+ ioc_err(mrioc, "ioc_facts timed out\n");
+ mpi3mr_check_rh_fault_ioc(mrioc,
MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
- mrioc->unrecoverable = 1;
retval = -1;
goto out_unlock;
}
@@ -2743,12 +2768,9 @@ static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
wait_for_completion_timeout(&mrioc->init_cmds.done,
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- mpi3mr_set_diagsave(mrioc);
- mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
+ mpi3mr_check_rh_fault_ioc(mrioc,
MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
- mrioc->unrecoverable = 1;
- ioc_err(mrioc, "Issue IOCInit: command timed out\n");
+ ioc_err(mrioc, "ioc_init timed out\n");
retval = -1;
goto out_unlock;
}
@@ -2839,12 +2861,9 @@ static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
wait_for_completion_timeout(&mrioc->init_cmds.done,
(MPI3MR_INTADMCMD_TIMEOUT * HZ));
if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
- mpi3mr_set_diagsave(mrioc);
- mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
+ ioc_err(mrioc, "event notification timed out\n");
+ mpi3mr_check_rh_fault_ioc(mrioc,
MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
- mrioc->unrecoverable = 1;
retval = -1;
goto out_unlock;
}
@@ -3051,29 +3070,28 @@ int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
goto out_unlock;
}
- if (!async) {
- wait_for_completion_timeout(&mrioc->init_cmds.done,
- (pe_timeout * HZ));
- if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
- ioc_err(mrioc, "Issue PortEnable: command timed out\n");
- retval = -1;
- mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
- mpi3mr_set_diagsave(mrioc);
- mpi3mr_issue_reset(mrioc,
- MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
- MPI3MR_RESET_FROM_PE_TIMEOUT);
- mrioc->unrecoverable = 1;
- goto out_unlock;
- }
- mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
+ if (async) {
+ mutex_unlock(&mrioc->init_cmds.mutex);
+ goto out;
}
+
+ wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));
+ if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
+ ioc_err(mrioc, "port enable timed out\n");
+ retval = -1;
+ mpi3mr_check_rh_fault_ioc(mrioc, MPI3MR_RESET_FROM_PE_TIMEOUT);
+ goto out_unlock;
+ }
+ mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
+
out_unlock:
+ mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
mutex_unlock(&mrioc->init_cmds.mutex);
out:
return retval;
}
-/* Protocol type to name mapper structure*/
+/* Protocol type to name mapper structure */
static const struct {
u8 protocol;
char *name;
--
2.27.0
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4218 bytes --]
next prev parent reply other threads:[~2021-12-20 14:04 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-12-20 14:11 [PATCH 00/25] mpi3mr: driver fixes and enhancements Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 01/25] mpi3mr: Add debug APIs based on logging_level bits Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 02/25] mpi3mr: replace spin_lock with spin_lock_irqsave Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 03/25] mpi3mr: Don't reset IOC if cmnds flush with reset status Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 04/25] mpi3mr: Update MPI3 headers - part1 Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 05/25] mpi3mr: Update MPI3 headers - part2 Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 06/25] mpi3mr: Add support for PCIe Managed Switch SES device Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 07/25] mpi3mr: Do access status validation before adding devices Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 08/25] mpi3mr: Increase internal cmnds timeout to 60s Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 09/25] mpi3mr: Handling unaligned PLL in unmap cmnds Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 10/25] mpi3mr: Display IOC firmware package version Sreekanth Reddy
2021-12-20 14:11 ` Sreekanth Reddy [this message]
2021-12-20 14:11 ` [PATCH 12/25] mpi3mr: code refactor of IOC init patch - part1 Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 13/25] mpi3mr: code refactor of IOC init patch - part2 Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 14/25] mpi3mr: Handle offline FW activation in graceful manner Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 15/25] mpi3mr: Add IOC reinit function Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 16/25] mpi3mr: Detect async reset occurred in firmware Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 17/25] mpi3mr: Gracefully handle online FW update operation Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 18/25] mpi3mr: Add Event acknowledgment logic Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 19/25] mpi3mr: Add support Prepare for Reset event Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 20/25] mpi3mr: Print cable mngnt and temp threshold events Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 21/25] mpi3mr: Add iouring interface support in io-polled mode Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 22/25] mpi3mr: use TM response codes from MPI3 headers Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 23/25] mpi3mr: Enhanced Task Management Support Reply handling Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 24/25] mpi3mr: Fixes around reply request queues Sreekanth Reddy
2021-12-20 14:11 ` [PATCH 25/25] mpi3mr: Bump driver version to 8.0.0.61.0 Sreekanth Reddy
2021-12-23 5:06 ` [PATCH 00/25] mpi3mr: driver fixes and enhancements Martin K. Petersen
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211220141159.16117-12-sreekanth.reddy@broadcom.com \
--to=sreekanth.reddy@broadcom.com \
--cc=linux-scsi@vger.kernel.org \
--cc=martin.petersen@oracle.com \
--cc=mpi3mr-linuxdrv.pdl@broadcom.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.