From: Kashyap Desai <kashyap.desai@broadcom.com>
To: linux-scsi@vger.kernel.org
Cc: jejb@linux.ibm.com, martin.petersen@oracle.com,
steve.hagan@broadcom.com, peter.rivera@broadcom.com,
mpi3mr-linuxdrv.pdl@broadcom.com,
Kashyap Desai <kashyap.desai@broadcom.com>,
sathya.prakash@broadcom.com
Subject: [PATCH v4 05/24] mpi3mr: add support of internal watchdog thread
Date: Wed, 12 May 2021 01:24:04 +0530 [thread overview]
Message-ID: <20210511195423.2134562-6-kashyap.desai@broadcom.com> (raw)
In-Reply-To: <20210511195423.2134562-1-kashyap.desai@broadcom.com>
[-- Attachment #1: Type: text/plain, Size: 7694 bytes --]
Watchdog thread is driver's internal thread which does few things like
detecting FW fault and reset the controller, Timestamp sync etc.
Signed-off-by: Kashyap Desai <kashyap.desai@broadcom.com>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Tomas Henzl <thenzl@redhat.com>
Reviewed-by: Himanshu Madhani <himanshu.madhani@oracle.com>
Cc: sathya.prakash@broadcom.com
---
drivers/scsi/mpi3mr/mpi3mr.h | 11 +++
drivers/scsi/mpi3mr/mpi3mr_fw.c | 125 ++++++++++++++++++++++++++++++++
drivers/scsi/mpi3mr/mpi3mr_os.c | 5 +-
3 files changed, 140 insertions(+), 1 deletion(-)
diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h
index 8804dd027086..0ff2b3b1947b 100644
--- a/drivers/scsi/mpi3mr/mpi3mr.h
+++ b/drivers/scsi/mpi3mr/mpi3mr.h
@@ -478,6 +478,10 @@ struct scmd_priv {
* @sense_buf_q_dma: Sense buffer queue DMA address
* @sbq_lock: Sense buffer queue lock
* @sbq_host_index: Sense buffer queuehost index
+ * @watchdog_work_q_name: Fault watchdog worker thread name
+ * @watchdog_work_q: Fault watchdog worker thread
+ * @watchdog_work: Fault watchdog work
+ * @watchdog_lock: Fault watchdog lock
* @is_driver_loading: Is driver still loading
* @scan_started: Async scan started
* @scan_failed: Asycn scan failed
@@ -491,6 +495,7 @@ struct scmd_priv {
* @chain_buf_lock: Chain buffer list lock
* @reset_in_progress: Reset in progress flag
* @unrecoverable: Controller unrecoverable flag
+ * @diagsave_timeout: Diagnostic information save timeout
* @logging_level: Controller debug logging level
* @current_event: Firmware event currently in process
* @driver_info: Driver, Kernel, OS information to firmware
@@ -572,6 +577,11 @@ struct mpi3mr_ioc {
spinlock_t sbq_lock;
u32 sbq_host_index;
+ char watchdog_work_q_name[20];
+ struct workqueue_struct *watchdog_work_q;
+ struct delayed_work watchdog_work;
+ spinlock_t watchdog_lock;
+
u8 is_driver_loading;
u8 scan_started;
u16 scan_failed;
@@ -589,6 +599,7 @@ struct mpi3mr_ioc {
u8 reset_in_progress;
u8 unrecoverable;
+ u16 diagsave_timeout;
int logging_level;
struct mpi3mr_fwevt *current_event;
diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
index 7779e2b965e6..9a750f630d5e 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_fw.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
@@ -1463,6 +1463,129 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
return retval;
}
+/**
+ * mpi3mr_watchdog_work - watchdog thread to monitor faults
+ * @work: work struct
+ *
+ * Watch dog work periodically executed (1 second interval) to
+ * monitor firmware fault and to issue periodic timer sync to
+ * the firmware.
+ *
+ * Return: Nothing.
+ */
+static void mpi3mr_watchdog_work(struct work_struct *work)
+{
+ struct mpi3mr_ioc *mrioc =
+ container_of(work, struct mpi3mr_ioc, watchdog_work.work);
+ unsigned long flags;
+ enum mpi3mr_iocstate ioc_state;
+ u32 fault, host_diagnostic;
+
+ /*Check for fault state every one second and issue Soft reset*/
+ ioc_state = mpi3mr_get_iocstate(mrioc);
+ if (ioc_state == MRIOC_STATE_FAULT) {
+ fault = readl(&mrioc->sysif_regs->fault) &
+ MPI3_SYSIF_FAULT_CODE_MASK;
+ host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
+ if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
+ if (!mrioc->diagsave_timeout) {
+ mpi3mr_print_fault_info(mrioc);
+ ioc_warn(mrioc, "Diag save in progress\n");
+ }
+ if ((mrioc->diagsave_timeout++) <=
+ MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
+ goto schedule_work;
+ } else
+ mpi3mr_print_fault_info(mrioc);
+ mrioc->diagsave_timeout = 0;
+
+ if (fault == MPI3_SYSIF_FAULT_CODE_FACTORY_RESET) {
+ ioc_info(mrioc,
+ "Factory Reset fault occurred marking controller as unrecoverable"
+ );
+ mrioc->unrecoverable = 1;
+ goto out;
+ }
+
+ if ((fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) ||
+ (fault == MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS) ||
+ (mrioc->reset_in_progress))
+ goto out;
+ if (fault == MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET)
+ mpi3mr_soft_reset_handler(mrioc,
+ MPI3MR_RESET_FROM_CIACTIV_FAULT, 0);
+ else
+ mpi3mr_soft_reset_handler(mrioc,
+ MPI3MR_RESET_FROM_FAULT_WATCH, 0);
+ }
+
+schedule_work:
+ spin_lock_irqsave(&mrioc->watchdog_lock, flags);
+ if (mrioc->watchdog_work_q)
+ queue_delayed_work(mrioc->watchdog_work_q,
+ &mrioc->watchdog_work,
+ msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
+ spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
+out:
+ return;
+}
+
+/**
+ * mpi3mr_start_watchdog - Start watchdog
+ * @mrioc: Adapter instance reference
+ *
+ * Create and start the watchdog thread to monitor controller
+ * faults.
+ *
+ * Return: Nothing.
+ */
+void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
+{
+ if (mrioc->watchdog_work_q)
+ return;
+
+ INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
+ snprintf(mrioc->watchdog_work_q_name,
+ sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
+ mrioc->id);
+ mrioc->watchdog_work_q =
+ create_singlethread_workqueue(mrioc->watchdog_work_q_name);
+ if (!mrioc->watchdog_work_q) {
+ ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
+ return;
+ }
+
+ if (mrioc->watchdog_work_q)
+ queue_delayed_work(mrioc->watchdog_work_q,
+ &mrioc->watchdog_work,
+ msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
+}
+
+/**
+ * mpi3mr_stop_watchdog - Stop watchdog
+ * @mrioc: Adapter instance reference
+ *
+ * Stop the watchdog thread created to monitor controller
+ * faults.
+ *
+ * Return: Nothing.
+ */
+void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
+{
+ unsigned long flags;
+ struct workqueue_struct *wq;
+
+ spin_lock_irqsave(&mrioc->watchdog_lock, flags);
+ wq = mrioc->watchdog_work_q;
+ mrioc->watchdog_work_q = NULL;
+ spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
+ if (wq) {
+ if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
+ flush_workqueue(wq);
+ destroy_workqueue(wq);
+ }
+}
+
/**
* mpi3mr_setup_admin_qpair - Setup admin queue pair
* @mrioc: Adapter instance reference
@@ -2609,6 +2732,8 @@ void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
{
enum mpi3mr_iocstate ioc_state;
+ mpi3mr_stop_watchdog(mrioc);
+
mpi3mr_ioc_disable_intr(mrioc);
ioc_state = mpi3mr_get_iocstate(mrioc);
diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c
index 7b3072e72c79..1587c8b029ad 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_os.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_os.c
@@ -281,7 +281,7 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
case MPI3_IOCSTATUS_SUCCESS:
scmd->result = (DID_OK << 16) | scsi_status;
if ((scsi_state & (MPI3_SCSI_STATE_NO_SCSI_STATUS)) ||
- (sense_state == MPI3_SCSI_STATE_SENSE_FAILED) ||
+ (sense_state == MPI3_SCSI_STATE_SENSE_FAILED) ||
(sense_state == MPI3_SCSI_STATE_SENSE_BUFF_Q_EMPTY))
scmd->result = DID_SOFT_ERROR << 16;
else if (scsi_state & MPI3_SCSI_STATE_TERMINATED)
@@ -568,6 +568,7 @@ static int mpi3mr_scan_finished(struct Scsi_Host *shost,
if (mrioc->scan_started)
return 0;
ioc_info(mrioc, "%s :port enable: SUCCESS\n", __func__);
+ mpi3mr_start_watchdog(mrioc);
mrioc->is_driver_loading = 0;
return 1;
@@ -852,9 +853,11 @@ mpi3mr_probe(struct pci_dev *pdev, const struct pci_device_id *id)
spin_lock_init(&mrioc->admin_req_lock);
spin_lock_init(&mrioc->reply_free_queue_lock);
spin_lock_init(&mrioc->sbq_lock);
+ spin_lock_init(&mrioc->watchdog_lock);
spin_lock_init(&mrioc->chain_buf_lock);
mpi3mr_init_drv_cmd(&mrioc->init_cmds, MPI3MR_HOSTTAG_INITCMDS);
+
if (pdev->revision)
mrioc->enable_segqueue = true;
--
2.18.1
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4212 bytes --]
next prev parent reply other threads:[~2021-05-11 19:51 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-11 19:53 [PATCH v4 00/24] Introducing mpi3mr driver Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 01/24] mpi3mr: add mpi30 Rev-R headers and Kconfig Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 02/24] mpi3mr: base driver code Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 03/24] mpi3mr: create operational request and reply queue pair Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 04/24] mpi3mr: add support of queue command processing Kashyap Desai
2021-05-11 19:54 ` Kashyap Desai [this message]
2021-05-11 19:54 ` [PATCH v4 06/24] mpi3mr: add support of event handling part-1 Kashyap Desai
2021-05-12 8:25 ` kernel test robot
2021-05-12 8:25 ` kernel test robot
2021-05-11 19:54 ` [PATCH v4 07/24] mpi3mr: add support of event handling pcie devices part-2 Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 08/24] mpi3mr: add support of event handling part-3 Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 09/24] mpi3mr: add support for recovering controller Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 10/24] mpi3mr: add support of timestamp sync with firmware Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 11/24] mpi3mr: print ioc info for debugging Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 12/24] mpi3mr: add bios_param shost template hook Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 13/24] mpi3mr: implement scsi error handler hooks Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 14/24] mpi3mr: add change queue depth support Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 15/24] mpi3mr: allow certain commands during pci-remove hook Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 16/24] mpi3mr: hardware workaround for UNMAP commands to nvme drives Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 17/24] mpi3mr: add support of threaded isr Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 18/24] mpi3mr: add complete support of soft reset Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 19/24] mpi3mr: print pending host ios for debug Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 20/24] mpi3mr: wait for pending IO completions upon detection of VD IO timeout Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 21/24] mpi3mr: add support of PM suspend and resume Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 22/24] mpi3mr: add support of DSN secure fw check Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 23/24] mpi3mr: add eedp dif dix support Kashyap Desai
2021-05-11 19:54 ` [PATCH v4 24/24] mpi3mr: add event handling debug prints Kashyap Desai
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210511195423.2134562-6-kashyap.desai@broadcom.com \
--to=kashyap.desai@broadcom.com \
--cc=jejb@linux.ibm.com \
--cc=linux-scsi@vger.kernel.org \
--cc=martin.petersen@oracle.com \
--cc=mpi3mr-linuxdrv.pdl@broadcom.com \
--cc=peter.rivera@broadcom.com \
--cc=sathya.prakash@broadcom.com \
--cc=steve.hagan@broadcom.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.