Linux-NVME Archive on lore.kernel.org
 help / color / Atom feed
From: Balbir Singh <bsingharora@gmail.com>
To: Keith Busch <kbusch@kernel.org>
Cc: Sagi Grimberg <sagi@grimberg.me>,
	James Smart <james.smart@broadcom.com>,
	linux-nvme@lists.infradead.org,
	Edmund Nadolski <edmund.nadolski@intel.com>,
	Judy Brock <judy.brock@samsung.com>,
	Christoph Hellwig <hch@lst.de>
Subject: Re: [PATCHv4 5/5] nvme: Wait for reset state when required
Date: Wed, 16 Oct 2019 11:56:57 +1100
Message-ID: <20191016005247.GA18485@balbir-desktop> (raw)
In-Reply-To: <20191010165736.12081-6-kbusch@kernel.org>

On Fri, Oct 11, 2019 at 01:57:36AM +0900, Keith Busch wrote:
> Prevent simultaneous controller disabling/enabling tasks from interfering
> with each other through a method to wait until the task successfully
> transitioned the controller to the RESETTING state. This ensures disabling
> the controller will not be interrupted by another reset path, otherwise
> a concurrent reset would leave the controller in the wrong state.
> 

Yes, I've run into this when nvme_core.io_timeout is smaller than the
time required to wait on reset (CTS.RDY) and multiple resets occur one
after the other.

> Signed-off-by: Keith Busch <kbusch@kernel.org>
> ---
>  drivers/nvme/host/core.c | 41 +++++++++++++++++++++++++++++++++--
>  drivers/nvme/host/nvme.h |  4 ++++
>  drivers/nvme/host/pci.c  | 46 +++++++++++++++++++++++++++-------------
>  3 files changed, 74 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index 0f5a82b7d3a4..7442c99a5ed7 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -126,7 +126,7 @@ static void nvme_queue_scan(struct nvme_ctrl *ctrl)
>   * code paths that can't be interrupted by other reset attempts. A hot removal
>   * may prevent this from succeeding.
>   */
> -static int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
> +int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
>  {
>  	if (ctrl->state != NVME_CTRL_RESETTING)
>  		return -EBUSY;
> @@ -134,6 +134,7 @@ static int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
>  		return -EBUSY;
>  	return 0;
>  }
> +EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
>  
>  int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
>  {
> @@ -384,8 +385,10 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>  		break;
>  	}
>  
> -	if (changed)
> +	if (changed) {
>  		ctrl->state = new_state;
> +		wake_up_all(&ctrl->state_wq);
> +	}

Shouldn't we wake up only if the new_state is terminal or if the
new_state is NVME_CTRL_RESETTING?

>  
>  	spin_unlock_irqrestore(&ctrl->lock, flags);
>  	if (changed && ctrl->state == NVME_CTRL_LIVE)
> @@ -394,6 +397,39 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>  }
>  EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
>  
> +/*
> + * Returns true for sink states that can't ever transition back to live.
> + */
> +static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
> +{
> +	switch (ctrl->state) {
> +	case NVME_CTRL_NEW:
> +	case NVME_CTRL_LIVE:
> +	case NVME_CTRL_RESETTING:
> +	case NVME_CTRL_CONNECTING:
> +		return false;
> +	case NVME_CTRL_DELETING:
> +	case NVME_CTRL_DEAD:
> +		return true;
> +	default:
> +		WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
> +		return true;
> +	}
> +}
> +
> +/*
> + * Waits for the controller state to be resetting, or returns false if it is
> + * not possible to ever transition to that state.
> + */
> +bool nvme_wait_reset(struct nvme_ctrl *ctrl)
> +{
> +	wait_event(ctrl->state_wq,
> +		   nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||

Would we expect this nvme_change_ctrl_state to cause a wake_up_all()
right away?

> +		   nvme_state_terminal(ctrl));
> +	return ctrl->state == NVME_CTRL_RESETTING;
> +}
> +EXPORT_SYMBOL_GPL(nvme_wait_reset);
> +
>  static void nvme_free_ns_head(struct kref *ref)
>  {
>  	struct nvme_ns_head *head =
> @@ -3999,6 +4035,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
>  	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
>  	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
>  	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
> +	init_waitqueue_head(&ctrl->state_wq);
>  
>  	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
>  	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
> diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
> index 2ba577271ada..22e8401352c2 100644
> --- a/drivers/nvme/host/nvme.h
> +++ b/drivers/nvme/host/nvme.h
> @@ -15,6 +15,7 @@
>  #include <linux/sed-opal.h>
>  #include <linux/fault-inject.h>
>  #include <linux/rcupdate.h>
> +#include <linux/wait.h>
>  
>  #include <trace/events/block.h>
>  
> @@ -198,6 +199,7 @@ struct nvme_ctrl {
>  	struct cdev cdev;
>  	struct work_struct reset_work;
>  	struct work_struct delete_work;
> +	wait_queue_head_t state_wq;
>  
>  	struct nvme_subsystem *subsys;
>  	struct list_head subsys_entry;
> @@ -448,6 +450,7 @@ void nvme_complete_rq(struct request *req);
>  bool nvme_cancel_request(struct request *req, void *data, bool reserved);
>  bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>  		enum nvme_ctrl_state new_state);
> +bool nvme_wait_reset(struct nvme_ctrl *ctrl);
>  int nvme_disable_ctrl(struct nvme_ctrl *ctrl);
>  int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
>  int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
> @@ -498,6 +501,7 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
>  void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
>  int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
>  int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
> +int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
>  int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
>  
>  int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
> index 7d0de87d733d..4fcfd01f350d 100644
> --- a/drivers/nvme/host/pci.c
> +++ b/drivers/nvme/host/pci.c
> @@ -2463,6 +2463,14 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
>  	mutex_unlock(&dev->shutdown_lock);
>  }
>  
> +static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
> +{

This sounds like we should disable the prepare reset, but I think it
really means in nvme disable, prepare for reset. Should we call it
nvme_disable_and_prepare_reset()?

I've not checked the state machine by hand, but

Acked-by: Balbir Singh <bsingharora@gmail.com>

_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

  reply index

Thread overview: 13+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-10-10 16:57 [PATCHv4 0/5] nvme: double reset prevention Keith Busch
2019-10-10 16:57 ` [PATCHv4 1/5] nvme-pci: Free tagset if no IO queues Keith Busch
2019-10-10 16:57 ` [PATCHv4 2/5] nvme: Remove ADMIN_ONLY state Keith Busch
2019-10-10 16:57 ` [PATCHv4 3/5] nvme: Restart request timers in resetting state Keith Busch
2019-10-10 16:57 ` [PATCHv4 4/5] nvme: Prevent resets during paused controller state Keith Busch
2019-10-10 16:57 ` [PATCHv4 5/5] nvme: Wait for reset state when required Keith Busch
2019-10-16  0:56   ` Balbir Singh [this message]
2019-10-10 17:09 ` [PATCHv4 0/5] nvme: double reset prevention Nadolski, Edmund
2019-10-11 15:58 ` Nadolski, Edmund
2019-10-11 16:27   ` Keith Busch
2019-10-14  7:15     ` Christoph Hellwig
2019-10-14 15:06       ` Keith Busch
2019-10-14  7:14 ` Christoph Hellwig

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20191016005247.GA18485@balbir-desktop \
    --to=bsingharora@gmail.com \
    --cc=edmund.nadolski@intel.com \
    --cc=hch@lst.de \
    --cc=james.smart@broadcom.com \
    --cc=judy.brock@samsung.com \
    --cc=kbusch@kernel.org \
    --cc=linux-nvme@lists.infradead.org \
    --cc=sagi@grimberg.me \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-NVME Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-nvme/0 linux-nvme/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-nvme linux-nvme/ https://lore.kernel.org/linux-nvme \
		linux-nvme@lists.infradead.org
	public-inbox-index linux-nvme

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.infradead.lists.linux-nvme


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git