draft ANA support v3

draft ANA support v3

[Christoph Hellwig]

The big TODO list item for the kernel code is support for transition
timeouts.

Besides that we still need nvme-cli and nvmetcli support as well as
an automated test suite.

A git tree is available at:

    git://git.infradead.org/users/hch/block.git nvme-ana

Gitweb:

    http://git.infradead.org/users/hch/block.git/shortlog/refs/heads/nvme-ana

Changes since v2:
 - support changing the state for the default ANA group
 - drop ANATT configuration again
 - delay gendisk registration until we have an I/O capable path
 - remove nvme_req_needs_failover and the SCT=3 handling in it,
   we actually do the right thing without it
 - refactor the get log page helpers

Changes since v1:
 - fix the check for SCT = 3
 - fix the check for immutable ANAGRPID
 - disable ANA if the ANA log can't be transferred in a single chunk
 - enable the ANA AEN explicitly
 - don't zero buffers that don't get copied to the SGL
 - use READ_ONCE/WRITE_ONCE for the ana_state fields
 - update ANA state on completion with an ANA status code
 - improve commit logs
 - add proper AEN enable and masking conditionals
 - fix the numeric value for the change state
 - fix a memory leak (Hannes Reinecke)
 - fix reporting of NUSE (based on a patch from Hannes Reinecke)
 - make ANATT configurable

[PATCH 01/10] nvme.h: add support for the log specific field

[Christoph Hellwig]

NVMe 1.3 added a new log specific field to the get log page CQ
defintion, add it to our get_log_page SQ structure.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 include/linux/nvme.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/linux/nvme.h b/include/linux/nvme.h
index 2950ce957656..32f24f4ca763 100644
--- a/include/linux/nvme.h
+++ b/include/linux/nvme.h
@@ -880,7 +880,7 @@ struct nvme_get_log_page_command {
 	__u64			rsvd2[2];
 	union nvme_data_ptr	dptr;
 	__u8			lid;
-	__u8			rsvd10;
+	__u8			lsp; /* upper 4 bits reserved */
 	__le16			numdl;
 	__le16			numdu;
 	__u16			rsvd11;
-- 
2.14.2

[PATCH 01/10] nvme.h: add support for the log specific field

[Johannes Thumshirn]

Looks good,
Reviewed-by: Johannes Thumshirn <jthumshirn at suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn at suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: Felix Imend?rffer, Jane Smithard, Graham Norton
HRB 21284 (AG N?rnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

[PATCH 01/10] nvme.h: add support for the log specific field

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:02 +0200
Christoph Hellwig <hch@lst.de> wrote:

> NVMe 1.3 added a new log specific field to the get log page CQ
> defintion, add it to our get_log_page SQ structure.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  include/linux/nvme.h | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/include/linux/nvme.h b/include/linux/nvme.h
> index 2950ce957656..32f24f4ca763 100644
> --- a/include/linux/nvme.h
> +++ b/include/linux/nvme.h
> @@ -880,7 +880,7 @@ struct nvme_get_log_page_command {
>  	__u64			rsvd2[2];
>  	union nvme_data_ptr	dptr;
>  	__u8			lid;
> -	__u8			rsvd10;
> +	__u8			lsp; /* upper 4 bits reserved */
>  	__le16			numdl;
>  	__le16			numdu;
>  	__u16			rsvd11;

Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 01/10] nvme.h: add support for the log specific field

[Sagi Grimberg]

Reviewed-by: Sagi Grimberg <sagi at grimberg.me>

[PATCH 02/10] nvme.h: add ANA definitions

[Christoph Hellwig]

Add various defintions from NVMe 1.3 TP 4004.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 include/linux/nvme.h | 50 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 47 insertions(+), 3 deletions(-)

diff --git a/include/linux/nvme.h b/include/linux/nvme.h
index 32f24f4ca763..fdff3c37bdd9 100644
--- a/include/linux/nvme.h
+++ b/include/linux/nvme.h
@@ -242,7 +242,12 @@ struct nvme_id_ctrl {
 	__le32			sanicap;
 	__le32			hmminds;
 	__le16			hmmaxd;
-	__u8			rsvd338[174];
+	__u8			rsvd338[4];
+	__u8			anatt;
+	__u8			anacap;
+	__le32			anagrpmax;
+	__le32			nanagrpid;
+	__u8			rsvd352[160];
 	__u8			sqes;
 	__u8			cqes;
 	__le16			maxcmd;
@@ -258,7 +263,8 @@ struct nvme_id_ctrl {
 	__le16			acwu;
 	__u8			rsvd534[2];
 	__le32			sgls;
-	__u8			rsvd540[228];
+	__le32			mnan;
+	__u8			rsvd544[224];
 	char			subnqn[256];
 	__u8			rsvd1024[768];
 	__le32			ioccsz;
@@ -312,7 +318,9 @@ struct nvme_id_ns {
 	__le16			nabspf;
 	__le16			noiob;
 	__u8			nvmcap[16];
-	__u8			rsvd64[40];
+	__u8			rsvd64[28];
+	__le32			anagrpid;
+	__u8			rsvd96[8];
 	__u8			nguid[16];
 	__u8			eui64[8];
 	struct nvme_lbaf	lbaf[16];
@@ -425,6 +433,32 @@ struct nvme_effects_log {
 	__u8   resv[2048];
 };
 
+enum nvme_ana_state {
+	NVME_ANA_OPTIMIZED		= 0x01,
+	NVME_ANA_NONOPTIMIZED		= 0x02,
+	NVME_ANA_INACCESSIBLE		= 0x03,
+	NVME_ANA_PERSISTENT_LOSS	= 0x04,
+	NVME_ANA_CHANGE			= 0x0f,
+};
+
+struct nvme_ana_group_desc {
+	__le32	grpid;
+	__le32	nnsids;
+	__le64	chgcnt;
+	__u8	state;
+	__u8	rsvd17[7];
+	__le32	nsids[];
+};
+
+/* flag for the log specific field of the ANA log */
+#define NVME_ANA_LOG_RGO	(1 << 0)
+
+struct nvme_ana_rsp_hdr {
+	__le64	chgcnt;
+	__le16	ngrps;
+	__le16	rsvd10[3];
+};
+
 enum {
 	NVME_SMART_CRIT_SPARE		= 1 << 0,
 	NVME_SMART_CRIT_TEMPERATURE	= 1 << 1,
@@ -444,11 +478,13 @@ enum {
 enum {
 	NVME_AER_NOTICE_NS_CHANGED	= 0x00,
 	NVME_AER_NOTICE_FW_ACT_STARTING = 0x01,
+	NVME_AER_NOTICE_ANA		= 0x03,
 };
 
 enum {
 	NVME_AEN_CFG_NS_ATTR		= 1 << 8,
 	NVME_AEN_CFG_FW_ACT		= 1 << 9,
+	NVME_AEN_CFG_ANA_CHANGE		= 1 << 11,
 };
 
 struct nvme_lba_range_type {
@@ -758,6 +794,7 @@ enum {
 	NVME_LOG_FW_SLOT	= 0x03,
 	NVME_LOG_CHANGED_NS	= 0x04,
 	NVME_LOG_CMD_EFFECTS	= 0x05,
+	NVME_LOG_ANA		= 0x0c,
 	NVME_LOG_DISC		= 0x70,
 	NVME_LOG_RESERVATION	= 0x80,
 	NVME_FWACT_REPL		= (0 << 3),
@@ -1180,6 +1217,13 @@ enum {
 	NVME_SC_ACCESS_DENIED		= 0x286,
 	NVME_SC_UNWRITTEN_BLOCK		= 0x287,
 
+	/*
+	 * Path-related Errors:
+	 */
+	NVME_SC_ANA_PERSISTENT_LOSS	= 0x301,
+	NVME_SC_ANA_INACCESSIBLE	= 0x302,
+	NVME_SC_ANA_TRANSITION		= 0x303,
+
 	NVME_SC_DNR			= 0x4000,
 };
 
-- 
2.14.2

[PATCH 02/10] nvme.h: add ANA definitions

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:03 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Add various defintions from NVMe 1.3 TP 4004.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  include/linux/nvme.h | 50
> +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47
> insertions(+), 3 deletions(-)
> 
Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 02/10] nvme.h: add ANA definitions

[Johannes Thumshirn]

Looks good,
Reviewed-by: Johannes Thumshirn <jthumshirn at suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn at suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: Felix Imend?rffer, Jane Smithard, Graham Norton
HRB 21284 (AG N?rnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

[PATCH 02/10] nvme.h: add ANA definitions

[Sagi Grimberg]

Reviewed-by: Sagi Grimberg <sagi at grimberg.me>

[PATCH 03/10] nvme: simplify the API for getting log pages

[Christoph Hellwig]

Merge nvme_get_log and nvme_get_log_ext into a single helper, which takes
a plain nsid instead of the nvme_ns pointer.  Also add support for the
log specific field while we're at it.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/host/core.c     | 32 +++++++++++---------------------
 drivers/nvme/host/lightnvm.c |  5 +++--
 drivers/nvme/host/nvme.h     |  4 ++--
 3 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 62d70419e12a..5e60cfeb81a9 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -2254,21 +2254,16 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 	return ret;
 }
 
-int nvme_get_log_ext(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
-		     u8 log_page, void *log,
-		     size_t size, u64 offset)
+int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
+		void *log, size_t size, u64 offset)
 {
 	struct nvme_command c = { };
 	unsigned long dwlen = size / 4 - 1;
 
 	c.get_log_page.opcode = nvme_admin_get_log_page;
-
-	if (ns)
-		c.get_log_page.nsid = cpu_to_le32(ns->head->ns_id);
-	else
-		c.get_log_page.nsid = cpu_to_le32(NVME_NSID_ALL);
-
+	c.get_log_page.nsid = cpu_to_le32(nsid);
 	c.get_log_page.lid = log_page;
+	c.get_log_page.lsp = lsp;
 	c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
 	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
 	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
@@ -2277,12 +2272,6 @@ int nvme_get_log_ext(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
 }
 
-static int nvme_get_log(struct nvme_ctrl *ctrl, u8 log_page, void *log,
-			size_t size)
-{
-	return nvme_get_log_ext(ctrl, NULL, log_page, log, size, 0);
-}
-
 static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
 {
 	int ret;
@@ -2293,8 +2282,8 @@ static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
 	if (!ctrl->effects)
 		return 0;
 
-	ret = nvme_get_log(ctrl, NVME_LOG_CMD_EFFECTS, ctrl->effects,
-					sizeof(*ctrl->effects));
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0,
+			ctrl->effects, sizeof(*ctrl->effects), 0);
 	if (ret) {
 		kfree(ctrl->effects);
 		ctrl->effects = NULL;
@@ -3213,7 +3202,8 @@ static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
 	 * raced with us in reading the log page, which could cause us to miss
 	 * updates.
 	 */
-	error = nvme_get_log(ctrl, NVME_LOG_CHANGED_NS, log, log_size);
+	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, log,
+			log_size, 0);
 	if (error)
 		dev_warn(ctrl->device,
 			"reading changed ns log failed: %d\n", error);
@@ -3330,9 +3320,9 @@ static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
 	if (!log)
 		return;
 
-	if (nvme_get_log(ctrl, NVME_LOG_FW_SLOT, log, sizeof(*log)))
-		dev_warn(ctrl->device,
-				"Get FW SLOT INFO log error\n");
+	if (nvme_get_log(ctrl, NVME_NSID_ALL, 0, NVME_LOG_FW_SLOT, log,
+			sizeof(*log), 0))
+		dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
 	kfree(log);
 }
 
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index 41279da799ed..5fc695e37a88 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -598,8 +598,9 @@ static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev,
 	while (left) {
 		len = min_t(unsigned int, left, ctrl->max_hw_sectors << 9);
 
-		ret = nvme_get_log_ext(ctrl, ns, NVME_NVM_LOG_REPORT_CHUNK,
-				dev_meta, len, offset);
+		ret = nvme_get_log(ctrl, ns->head->ns_id,
+				NVME_NVM_LOG_REPORT_CHUNK, 0, dev_meta, len,
+				offset);
 		if (ret) {
 			dev_err(ctrl->device, "Get REPORT CHUNK log error\n");
 			break;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index de24fe77c80b..0b0a14ce0ab5 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -437,8 +437,8 @@ int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
 int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
 int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
 
-int nvme_get_log_ext(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
-		u8 log_page, void *log, size_t size, u64 offset);
+int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
+		void *log, size_t size, u64 offset);
 
 extern const struct attribute_group nvme_ns_id_attr_group;
 extern const struct block_device_operations nvme_ns_head_ops;
-- 
2.14.2

[PATCH 03/10] nvme: simplify the API for getting log pages

[Johannes Thumshirn]

Looks good,
Reviewed-by: Johannes Thumshirn <jthumshirn at suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn at suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: Felix Imend?rffer, Jane Smithard, Graham Norton
HRB 21284 (AG N?rnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

[PATCH 03/10] nvme: simplify the API for getting log pages

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:04 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Merge nvme_get_log and nvme_get_log_ext into a single helper, which
> takes a plain nsid instead of the nvme_ns pointer.  Also add support
> for the log specific field while we're at it.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  drivers/nvme/host/core.c     | 32 +++++++++++---------------------
>  drivers/nvme/host/lightnvm.c |  5 +++--
>  drivers/nvme/host/nvme.h     |  4 ++--
>  3 files changed, 16 insertions(+), 25 deletions(-)
> 
Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 03/10] nvme: simplify the API for getting log pages

[Sagi Grimberg]

Reviewed-by: Sagi Grimberg <sagi at grimber.me>

[PATCH 04/10] nvme: remove nvme_req_needs_failover

[Christoph Hellwig]

Now that we just call out to blk_path_error there isn't really any good
reason to not merge it into the only caller.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/host/core.c      | 3 ++-
 drivers/nvme/host/multipath.c | 7 -------
 drivers/nvme/host/nvme.h      | 6 ------
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 5e60cfeb81a9..2fb284d0f497 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -236,7 +236,8 @@ void nvme_complete_rq(struct request *req)
 	trace_nvme_complete_rq(req);
 
 	if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
-		if (nvme_req_needs_failover(req, status)) {
+		if ((req->cmd_flags & REQ_NVME_MPATH) &&
+		    blk_path_error(status)) {
 			nvme_failover_req(req);
 			return;
 		}
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index d7b664ae5923..7996b98befb1 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -55,13 +55,6 @@ void nvme_failover_req(struct request *req)
 	kblockd_schedule_work(&ns->head->requeue_work);
 }
 
-bool nvme_req_needs_failover(struct request *req, blk_status_t error)
-{
-	if (!(req->cmd_flags & REQ_NVME_MPATH))
-		return false;
-	return blk_path_error(error);
-}
-
 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 {
 	struct nvme_ns *ns;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 0b0a14ce0ab5..6e7f59ee79dd 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -447,7 +447,6 @@ extern const struct block_device_operations nvme_ns_head_ops;
 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
 			struct nvme_ctrl *ctrl, int *flags);
 void nvme_failover_req(struct request *req);
-bool nvme_req_needs_failover(struct request *req, blk_status_t error);
 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
 void nvme_mpath_add_disk(struct nvme_ns_head *head);
@@ -484,11 +483,6 @@ static inline void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
 static inline void nvme_failover_req(struct request *req)
 {
 }
-static inline bool nvme_req_needs_failover(struct request *req,
-					   blk_status_t error)
-{
-	return false;
-}
 static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 {
 }
-- 
2.14.2

[PATCH 04/10] nvme: remove nvme_req_needs_failover

[Johannes Thumshirn]

Looks good,
Reviewed-by: Johannes Thumshirn <jthumshirn at suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn at suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: Felix Imend?rffer, Jane Smithard, Graham Norton
HRB 21284 (AG N?rnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

[PATCH 04/10] nvme: remove nvme_req_needs_failover

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:05 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Now that we just call out to blk_path_error there isn't really any
> good reason to not merge it into the only caller.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  drivers/nvme/host/core.c      | 3 ++-
>  drivers/nvme/host/multipath.c | 7 -------
>  drivers/nvme/host/nvme.h      | 6 ------
>  3 files changed, 2 insertions(+), 14 deletions(-)
> 
Hmm. Have to redo my testing, then; this was one area which turned out
to be quite tricky. But anyway:

Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 04/10] nvme: remove nvme_req_needs_failover

[Christoph Hellwig]

On Thu, Jun 07, 2018@10:01:11AM +0200, Hannes Reinecke wrote:
> On Wed,  6 Jun 2018 16:33:05 +0200
> Christoph Hellwig <hch@lst.de> wrote:
> 
> > Now that we just call out to blk_path_error there isn't really any
> > good reason to not merge it into the only caller.
> > 
> > Signed-off-by: Christoph Hellwig <hch at lst.de>
> > ---
> >  drivers/nvme/host/core.c      | 3 ++-
> >  drivers/nvme/host/multipath.c | 7 -------
> >  drivers/nvme/host/nvme.h      | 6 ------
> >  3 files changed, 2 insertions(+), 14 deletions(-)
> > 
> Hmm. Have to redo my testing, then; this was one area which turned out
> to be quite tricky. But anyway:

This patch does not change the logic at all, it just removes an inline
function that is no needed anymore.

[PATCH 04/10] nvme: remove nvme_req_needs_failover

[Sagi Grimberg]

Reviewed-by: Sagi Grimberg <sagi at grimberg.me>

[PATCH 05/10] nvme: add ANA support

[Christoph Hellwig]

Add support for Asynchronous Namespace Access as specified in NVMe 1.3
TP 4004.  With ANA each namespace attached to a controller belongs to
an ANA group that describes the characteristics of accessing the
namespaces through this controller.  In the optimized and non-optimized
states namespaces can be accessed regularly, although in a multi-pathing
environment we should always prefer to access a namespace through a
controller where an optimized relationship exists.  Namespaces in
Inaccessible, Permanent-Loss or Change state for a given controller
should not be accessed.

We keep a simple per-controller array of ANA states which is indexed
by the ANA group ID specified in the namespace.  The states are updated
through reading the ANA log page, which is read once during controller
initialization, and whenever the ANA change notice AEN is received, or
when one of the ANA specific status codes that signal a state change
is received on a command.

There currently isn't any support for the ANA transition timeout
yet.

Includes fixes and improvements from Hannes Reinecke.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/host/core.c      |  30 +++++-
 drivers/nvme/host/multipath.c | 211 ++++++++++++++++++++++++++++++++++++++++--
 drivers/nvme/host/nvme.h      |  28 ++++++
 3 files changed, 262 insertions(+), 7 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 2fb284d0f497..67bd73a98a1b 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1041,7 +1041,7 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
 
 #define NVME_AEN_SUPPORTED \
-	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT)
+	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | NVME_AEN_CFG_ANA_CHANGE)
 
 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
 {
@@ -1472,6 +1472,7 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
 	ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
 	if (ns->lba_shift == 0)
 		ns->lba_shift = 9;
+	ns->anagrpid = le32_to_cpu(id->anagrpid);
 	ns->noiob = le16_to_cpu(id->noiob);
 	ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
 	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
@@ -2375,6 +2376,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 	nvme_set_queue_limits(ctrl, ctrl->admin_q);
 	ctrl->sgls = le32_to_cpu(id->sgls);
 	ctrl->kas = le16_to_cpu(id->kas);
+	ctrl->max_namespaces = le32_to_cpu(id->mnan);
+	ctrl->anacap = id->anacap;
+	ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
+	ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 
 	if (id->rtd3e) {
 		/* us -> s */
@@ -2453,6 +2458,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 	if (ret < 0)
 		return ret;
 
+	ret = nvme_configure_ana(ctrl);
+	if (ret < 0)
+		return ret;
+
 	ctrl->identified = true;
 
 	return 0;
@@ -2654,6 +2663,10 @@ static struct attribute *nvme_ns_id_attrs[] = {
 	&dev_attr_nguid.attr,
 	&dev_attr_eui.attr,
 	&dev_attr_nsid.attr,
+#ifdef CONFIG_NVME_MULTIPATH
+	&dev_attr_ana_grpid.attr,
+	&dev_attr_ana_state.attr,
+#endif
 	NULL,
 };
 
@@ -2676,6 +2689,14 @@ static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
 		if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
 			return 0;
 	}
+#ifdef CONFIG_NVME_MULTIPATH
+	if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
+		if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
+			return 0;
+		if (!nvme_get_ns_from_dev(dev)->anagrpid)
+			return 0;
+	}
+#endif
 	return a->mode;
 }
 
@@ -3369,6 +3390,11 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
 	case NVME_AER_NOTICE_FW_ACT_STARTING:
 		queue_work(nvme_wq, &ctrl->fw_act_work);
 		break;
+	case NVME_AER_NOTICE_ANA:
+		if (WARN_ON_ONCE(!ctrl->ana_log_buf))
+			break;
+		queue_work(nvme_wq, &ctrl->ana_work);
+		break;
 	default:
 		dev_warn(ctrl->device, "async event result %08x\n", result);
 	}
@@ -3404,6 +3430,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
 	nvme_stop_keep_alive(ctrl);
 	flush_work(&ctrl->async_event_work);
 	flush_work(&ctrl->scan_work);
+	cancel_work_sync(&ctrl->ana_work);
 	cancel_work_sync(&ctrl->fw_act_work);
 	if (ctrl->ops->stop_ctrl)
 		ctrl->ops->stop_ctrl(ctrl);
@@ -3438,6 +3465,7 @@ static void nvme_free_ctrl(struct device *dev)
 
 	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
 	kfree(ctrl->effects);
+	nvme_deconfigure_ana(ctrl);
 
 	if (subsys) {
 		mutex_lock(&subsys->lock);
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 7996b98befb1..2ab55ca15cec 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 Christoph Hellwig.
+ * Copyright (c) 2017-2018 Christoph Hellwig.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -41,6 +41,11 @@ void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
 	}
 }
 
+static void nvme_update_ana_state(struct nvme_ns *ns, enum nvme_ana_state state)
+{
+	WRITE_ONCE(ns->ctrl->ana_state[ns->anagrpid], state);
+}
+
 void nvme_failover_req(struct request *req)
 {
 	struct nvme_ns *ns = req->q->queuedata;
@@ -51,7 +56,31 @@ void nvme_failover_req(struct request *req)
 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
 	blk_mq_end_request(req, 0);
 
-	nvme_reset_ctrl(ns->ctrl);
+	/*
+	 * Reset the controller for any non-ANA error as we don't know what
+	 * caused the error:
+	 */
+	switch (nvme_req(req)->status & 0x7ff) {
+	case NVME_SC_ANA_TRANSITION:
+		/*
+		 * XXX: We should verify the controller doesn't die on during
+		 * the transition.  But that means we per-group timeout from
+		 * when we first hit the change state, so this won't be
+		 * entirely trivial..
+		 */
+		nvme_update_ana_state(ns, NVME_ANA_CHANGE);
+		break;
+	case NVME_SC_ANA_PERSISTENT_LOSS:
+		nvme_update_ana_state(ns, NVME_ANA_PERSISTENT_LOSS);
+		break;
+	case NVME_SC_ANA_INACCESSIBLE:
+		nvme_update_ana_state(ns, NVME_ANA_INACCESSIBLE);
+		break;
+	default:
+		nvme_reset_ctrl(ns->ctrl);
+		break;
+	}
+
 	kblockd_schedule_work(&ns->head->requeue_work);
 }
 
@@ -67,12 +96,32 @@ void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 	up_read(&ctrl->namespaces_rwsem);
 }
 
-static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head)
+static inline enum nvme_ana_state nvme_ns_ana_state(struct nvme_ns *ns)
+{
+	if (!nvme_ctrl_has_ana(ns->ctrl))
+		return NVME_ANA_OPTIMIZED;
+	if (WARN_ON_ONCE(ns->anagrpid > ns->ctrl->anagrpmax))
+		return 0;
+	return READ_ONCE(ns->ctrl->ana_state[ns->anagrpid]);
+}
+
+static const char *nvme_ana_state_names[] = {
+	[0]				= "invalid state",
+	[NVME_ANA_OPTIMIZED]		= "optimized",
+	[NVME_ANA_NONOPTIMIZED]		= "non-optimized",
+	[NVME_ANA_INACCESSIBLE]		= "inaccessible",
+	[NVME_ANA_PERSISTENT_LOSS]	= "persistent-loss",
+	[NVME_ANA_CHANGE]		= "change",
+};
+
+static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head,
+		u8 ana_state)
 {
 	struct nvme_ns *ns;
 
 	list_for_each_entry_rcu(ns, &head->list, siblings) {
-		if (ns->ctrl->state == NVME_CTRL_LIVE) {
+		if (ns->ctrl->state == NVME_CTRL_LIVE &&
+		    nvme_ns_ana_state(ns) == ana_state) {
 			rcu_assign_pointer(head->current_path, ns);
 			return ns;
 		}
@@ -85,8 +134,14 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 {
 	struct nvme_ns *ns = srcu_dereference(head->current_path, &head->srcu);
 
-	if (unlikely(!ns || ns->ctrl->state != NVME_CTRL_LIVE))
-		ns = __nvme_find_path(head);
+	if (likely(ns && ns->ctrl->state == NVME_CTRL_LIVE &&
+			nvme_ns_ana_state(ns) == NVME_ANA_OPTIMIZED))
+		return ns;
+
+	ns = __nvme_find_path(head, NVME_ANA_OPTIMIZED);
+	if (!ns)
+		ns = __nvme_find_path(head, NVME_ANA_NONOPTIMIZED);
+	/* XXX: try an inaccessible path as last resort per 8.18.3.3 */
 	return ns;
 }
 
@@ -239,3 +294,147 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 	blk_cleanup_queue(head->disk->queue);
 	put_disk(head->disk);
 }
+
+static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
+{
+	void *base = ctrl->ana_log_buf;
+	struct nvme_ns *ns;
+	size_t offset;
+	int error, i;
+
+	/*
+	 * If anagrpid never changes we don't need to process the namespace
+	 * lists.
+	 */
+	if (ctrl->anacap & (1 << 6))
+		groups_only = true;
+
+	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
+			groups_only ? NVME_ANA_LOG_RGO : 0,
+			ctrl->ana_log_buf, ctrl->ana_log_size, 0);
+	if (error) {
+		dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
+		return error;
+	}
+
+	offset = sizeof(struct nvme_ana_rsp_hdr);
+	for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
+		struct nvme_ana_group_desc *desc = base + offset;
+		u32 grpid = le32_to_cpu(desc->grpid);
+		u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
+		size_t nsid_buf_size = nr_nsids * sizeof(__le32);
+
+		if (WARN_ON_ONCE(grpid == 0))
+			return -EINVAL;
+		if (WARN_ON_ONCE(grpid > ctrl->anagrpmax))
+			return -EINVAL;
+		if (WARN_ON_ONCE(desc->state == 0))
+			return -EINVAL;
+		if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
+			return -EINVAL;
+
+		dev_info(ctrl->device, "ANA group %d: %s.\n",
+				grpid, nvme_ana_state_names[desc->state]);
+		WRITE_ONCE(ctrl->ana_state[grpid], desc->state);
+		offset += sizeof(*desc);
+		if (!nr_nsids)
+			continue;
+
+		if (WARN_ON_ONCE(groups_only))
+			return -EINVAL;
+		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
+			return -EINVAL;
+
+		down_write(&ctrl->namespaces_rwsem);
+		list_for_each_entry(ns, &ctrl->namespaces, list) {
+			u32 nsid = le32_to_cpu(desc->nsids[n]);
+
+			if (ns->head->ns_id != nsid)
+				continue;
+			ns->anagrpid = grpid;
+			if (++n == nr_nsids)
+				break;
+		}
+		up_write(&ctrl->namespaces_rwsem);
+		WARN_ON_ONCE(n < nr_nsids);
+
+		offset += nsid_buf_size;
+		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void nvme_ana_work(struct work_struct *work)
+{
+	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
+
+	nvme_process_ana_log(ctrl, false);
+	nvme_kick_requeue_lists(ctrl);
+}
+
+int nvme_configure_ana(struct nvme_ctrl *ctrl)
+{
+	int error;
+
+	if (!nvme_ctrl_has_ana(ctrl))
+		return 0;
+
+	ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
+		ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
+	if (!(ctrl->anacap & (1 << 6)))
+		ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
+
+	if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
+		dev_err(ctrl->device,
+			"ANA log page size (%zd) larger than MDTS (%d).\n",
+			ctrl->ana_log_size,
+			ctrl->max_hw_sectors << SECTOR_SHIFT);
+		dev_err(ctrl->device, "disabling ANA support.\n");
+		return 0;
+	}
+
+	INIT_WORK(&ctrl->ana_work, nvme_ana_work);
+	ctrl->ana_state = kcalloc(ctrl->anagrpmax, sizeof(*ctrl->ana_state),
+			GFP_KERNEL);
+	if (!ctrl->ana_state)
+		return -ENOMEM;
+
+	ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
+	if (!ctrl->ana_log_buf)
+		goto out_free_ana_state;
+
+	error = nvme_process_ana_log(ctrl, true);
+	if (error)
+		goto out_free_ana_log_buf;
+	return 0;
+out_free_ana_log_buf:
+	kfree(ctrl->ana_log_buf);
+out_free_ana_state:
+	kfree(ctrl->ana_state);
+	return -ENOMEM;
+}
+
+void nvme_deconfigure_ana(struct nvme_ctrl *ctrl)
+{
+	kfree(ctrl->ana_log_buf);
+	kfree(ctrl->ana_state);
+}
+
+static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
+		char *buf)
+{
+	return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->anagrpid);
+}
+DEVICE_ATTR_RO(ana_grpid);
+
+static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
+		char *buf)
+{
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
+	enum nvme_ana_state state = nvme_ns_ana_state(ns);
+
+	return sprintf(buf, "%s\n", nvme_ana_state_names[state]);
+}
+DEVICE_ATTR_RO(ana_state);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 6e7f59ee79dd..82a58bd2bf8e 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -174,6 +174,7 @@ struct nvme_ctrl {
 	u16 oacs;
 	u16 nssa;
 	u16 nr_streams;
+	u32 max_namespaces;
 	atomic_t abort_limit;
 	u8 vwc;
 	u32 vs;
@@ -197,6 +198,15 @@ struct nvme_ctrl {
 #define EVENT_NS_CHANGED		(1 << 0)
 	unsigned long events;
 
+	/* asymmetric namespace access: */
+	u8 anacap;
+	u32 anagrpmax;
+	u32 nanagrpid;
+	enum nvme_ana_state *ana_state;
+	size_t ana_log_size;
+	struct nvme_ana_rsp_hdr *ana_log_buf;
+	struct work_struct ana_work;
+
 	/* Power saving configuration */
 	u64 ps_max_latency_us;
 	bool apst_enabled;
@@ -302,6 +312,7 @@ struct nvme_ns {
 #define NVME_NS_REMOVING 0
 #define NVME_NS_DEAD     1
 	u16 noiob;
+	u32 anagrpid;
 
 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
 	struct nvme_fault_inject fault_inject;
@@ -443,6 +454,11 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
 extern const struct attribute_group nvme_ns_id_attr_group;
 extern const struct block_device_operations nvme_ns_head_ops;
 
+static inline bool nvme_ctrl_has_ana(struct nvme_ctrl *ctrl)
+{
+	return ctrl->subsys->cmic & (1 << 3);
+}
+
 #ifdef CONFIG_NVME_MULTIPATH
 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
 			struct nvme_ctrl *ctrl, int *flags);
@@ -451,6 +467,8 @@ void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
 void nvme_mpath_add_disk(struct nvme_ns_head *head);
 void nvme_mpath_remove_disk(struct nvme_ns_head *head);
+int nvme_configure_ana(struct nvme_ctrl *ctrl);
+void nvme_deconfigure_ana(struct nvme_ctrl *ctrl);
 
 static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
 {
@@ -469,6 +487,9 @@ static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
 		kblockd_schedule_work(&head->requeue_work);
 }
 
+extern struct device_attribute dev_attr_ana_grpid;
+extern struct device_attribute dev_attr_ana_state;
+
 #else
 /*
  * Without the multipath code enabled, multiple controller per subsystems are
@@ -503,6 +524,13 @@ static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
 static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
 {
 }
+static inline int nvme_configure_ana(struct nvme_ctrl *ctrl)
+{
+	return 0;
+}
+static inline void nvme_deconfigure_ana(struct nvme_ctrl *ctrl)
+{
+}
 #endif /* CONFIG_NVME_MULTIPATH */
 
 #ifdef CONFIG_NVM
-- 
2.14.2

[PATCH 05/10] nvme: add ANA support

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:06 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Add support for Asynchronous Namespace Access as specified in NVMe 1.3
> TP 4004.  With ANA each namespace attached to a controller belongs to
> an ANA group that describes the characteristics of accessing the
> namespaces through this controller.  In the optimized and
> non-optimized states namespaces can be accessed regularly, although
> in a multi-pathing environment we should always prefer to access a
> namespace through a controller where an optimized relationship
> exists.  Namespaces in Inaccessible, Permanent-Loss or Change state
> for a given controller should not be accessed.
> 
> We keep a simple per-controller array of ANA states which is indexed
> by the ANA group ID specified in the namespace.  The states are
> updated through reading the ANA log page, which is read once during
> controller initialization, and whenever the ANA change notice AEN is
> received, or when one of the ANA specific status codes that signal a
> state change is received on a command.
> 
> There currently isn't any support for the ANA transition timeout
> yet.
> 
> Includes fixes and improvements from Hannes Reinecke.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  drivers/nvme/host/core.c      |  30 +++++-
>  drivers/nvme/host/multipath.c | 211
> ++++++++++++++++++++++++++++++++++++++++--
> drivers/nvme/host/nvme.h      |  28 ++++++ 3 files changed, 262
> insertions(+), 7 deletions(-)
> 

Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 05/10] nvme: add ANA support

[Sagi Grimberg]

On 06/06/2018 05:33 PM, Christoph Hellwig wrote:
> Add support for Asynchronous Namespace Access as specified in NVMe 1.3
> TP 4004.  With ANA each namespace attached to a controller belongs to
> an ANA group that describes the characteristics of accessing the
> namespaces through this controller.  In the optimized and non-optimized
> states namespaces can be accessed regularly, although in a multi-pathing
> environment we should always prefer to access a namespace through a
> controller where an optimized relationship exists.  Namespaces in
> Inaccessible, Permanent-Loss or Change state for a given controller
> should not be accessed.
> 
> We keep a simple per-controller array of ANA states which is indexed
> by the ANA group ID specified in the namespace.  The states are updated
> through reading the ANA log page, which is read once during controller
> initialization, and whenever the ANA change notice AEN is received, or
> when one of the ANA specific status codes that signal a state change
> is received on a command.
> 
> There currently isn't any support for the ANA transition timeout
> yet.
> 
> Includes fixes and improvements from Hannes Reinecke.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>   drivers/nvme/host/core.c      |  30 +++++-
>   drivers/nvme/host/multipath.c | 211 ++++++++++++++++++++++++++++++++++++++++--
>   drivers/nvme/host/nvme.h      |  28 ++++++
>   3 files changed, 262 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index 2fb284d0f497..67bd73a98a1b 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -1041,7 +1041,7 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
>   EXPORT_SYMBOL_GPL(nvme_set_queue_count);
>   
>   #define NVME_AEN_SUPPORTED \
> -	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT)
> +	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | NVME_AEN_CFG_ANA_CHANGE)
>   
>   static void nvme_enable_aen(struct nvme_ctrl *ctrl)
>   {
> @@ -1472,6 +1472,7 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
>   	ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
>   	if (ns->lba_shift == 0)
>   		ns->lba_shift = 9;
> +	ns->anagrpid = le32_to_cpu(id->anagrpid);
>   	ns->noiob = le16_to_cpu(id->noiob);
>   	ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
>   	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
> @@ -2375,6 +2376,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
>   	nvme_set_queue_limits(ctrl, ctrl->admin_q);
>   	ctrl->sgls = le32_to_cpu(id->sgls);
>   	ctrl->kas = le16_to_cpu(id->kas);
> +	ctrl->max_namespaces = le32_to_cpu(id->mnan);
> +	ctrl->anacap = id->anacap;
> +	ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
> +	ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
>   
>   	if (id->rtd3e) {
>   		/* us -> s */
> @@ -2453,6 +2458,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
>   	if (ret < 0)
>   		return ret;
>   
> +	ret = nvme_configure_ana(ctrl);
> +	if (ret < 0)
> +		return ret;
> +
>   	ctrl->identified = true;
>   
>   	return 0;
> @@ -2654,6 +2663,10 @@ static struct attribute *nvme_ns_id_attrs[] = {
>   	&dev_attr_nguid.attr,
>   	&dev_attr_eui.attr,
>   	&dev_attr_nsid.attr,
> +#ifdef CONFIG_NVME_MULTIPATH
> +	&dev_attr_ana_grpid.attr,
> +	&dev_attr_ana_state.attr,
> +#endif
>   	NULL,
>   };
>   
> @@ -2676,6 +2689,14 @@ static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
>   		if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
>   			return 0;
>   	}
> +#ifdef CONFIG_NVME_MULTIPATH
> +	if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
> +		if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
> +			return 0;
> +		if (!nvme_get_ns_from_dev(dev)->anagrpid)
> +			return 0;
> +	}
> +#endif
>   	return a->mode;
>   }
>   
> @@ -3369,6 +3390,11 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
>   	case NVME_AER_NOTICE_FW_ACT_STARTING:
>   		queue_work(nvme_wq, &ctrl->fw_act_work);
>   		break;
> +	case NVME_AER_NOTICE_ANA:
> +		if (WARN_ON_ONCE(!ctrl->ana_log_buf))
> +			break;
> +		queue_work(nvme_wq, &ctrl->ana_work);
> +		break;
>   	default:
>   		dev_warn(ctrl->device, "async event result %08x\n", result);
>   	}
> @@ -3404,6 +3430,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
>   	nvme_stop_keep_alive(ctrl);
>   	flush_work(&ctrl->async_event_work);
>   	flush_work(&ctrl->scan_work);
> +	cancel_work_sync(&ctrl->ana_work);
>   	cancel_work_sync(&ctrl->fw_act_work);
>   	if (ctrl->ops->stop_ctrl)
>   		ctrl->ops->stop_ctrl(ctrl);
> @@ -3438,6 +3465,7 @@ static void nvme_free_ctrl(struct device *dev)
>   
>   	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
>   	kfree(ctrl->effects);
> +	nvme_deconfigure_ana(ctrl);
>   
>   	if (subsys) {
>   		mutex_lock(&subsys->lock);
> diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
> index 7996b98befb1..2ab55ca15cec 100644
> --- a/drivers/nvme/host/multipath.c
> +++ b/drivers/nvme/host/multipath.c
> @@ -1,5 +1,5 @@
>   /*
> - * Copyright (c) 2017 Christoph Hellwig.
> + * Copyright (c) 2017-2018 Christoph Hellwig.
>    *
>    * This program is free software; you can redistribute it and/or modify it
>    * under the terms and conditions of the GNU General Public License,
> @@ -41,6 +41,11 @@ void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
>   	}
>   }
>   
> +static void nvme_update_ana_state(struct nvme_ns *ns, enum nvme_ana_state state)
> +{
> +	WRITE_ONCE(ns->ctrl->ana_state[ns->anagrpid], state);
> +}
> +
>   void nvme_failover_req(struct request *req)
>   {
>   	struct nvme_ns *ns = req->q->queuedata;
> @@ -51,7 +56,31 @@ void nvme_failover_req(struct request *req)
>   	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
>   	blk_mq_end_request(req, 0);
>   
> -	nvme_reset_ctrl(ns->ctrl);
> +	/*
> +	 * Reset the controller for any non-ANA error as we don't know what
> +	 * caused the error:
> +	 */
> +	switch (nvme_req(req)->status & 0x7ff) {
> +	case NVME_SC_ANA_TRANSITION:
> +		/*
> +		 * XXX: We should verify the controller doesn't die on during
> +		 * the transition.  But that means we per-group timeout from
> +		 * when we first hit the change state, so this won't be
> +		 * entirely trivial..

something is wrong with the sentence I think "that means we per-group
timeout from when..."

Not sure what you are trying to say.

> +		 */
> +		nvme_update_ana_state(ns, NVME_ANA_CHANGE);
> +		break;
> +	case NVME_SC_ANA_PERSISTENT_LOSS:
> +		nvme_update_ana_state(ns, NVME_ANA_PERSISTENT_LOSS);
> +		break;
> +	case NVME_SC_ANA_INACCESSIBLE:
> +		nvme_update_ana_state(ns, NVME_ANA_INACCESSIBLE);
> +		break;
> +	default:
> +		nvme_reset_ctrl(ns->ctrl);
> +		break;
> +	}
> +
>   	kblockd_schedule_work(&ns->head->requeue_work);
>   }
>   
> @@ -67,12 +96,32 @@ void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
>   	up_read(&ctrl->namespaces_rwsem);
>   }
>   
> -static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head)
> +static inline enum nvme_ana_state nvme_ns_ana_state(struct nvme_ns *ns)
> +{
> +	if (!nvme_ctrl_has_ana(ns->ctrl))
> +		return NVME_ANA_OPTIMIZED;
> +	if (WARN_ON_ONCE(ns->anagrpid > ns->ctrl->anagrpmax))
> +		return 0;
> +	return READ_ONCE(ns->ctrl->ana_state[ns->anagrpid]);
> +}
> +
> +static const char *nvme_ana_state_names[] = {
> +	[0]				= "invalid state",
> +	[NVME_ANA_OPTIMIZED]		= "optimized",
> +	[NVME_ANA_NONOPTIMIZED]		= "non-optimized",
> +	[NVME_ANA_INACCESSIBLE]		= "inaccessible",
> +	[NVME_ANA_PERSISTENT_LOSS]	= "persistent-loss",
> +	[NVME_ANA_CHANGE]		= "change",
> +};
> +
> +static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head,
> +		u8 ana_state)
>   {
>   	struct nvme_ns *ns;
>   
>   	list_for_each_entry_rcu(ns, &head->list, siblings) {
> -		if (ns->ctrl->state == NVME_CTRL_LIVE) {
> +		if (ns->ctrl->state == NVME_CTRL_LIVE &&
> +		    nvme_ns_ana_state(ns) == ana_state) {
>   			rcu_assign_pointer(head->current_path, ns);
>   			return ns;
>   		}
> @@ -85,8 +134,14 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
>   {
>   	struct nvme_ns *ns = srcu_dereference(head->current_path, &head->srcu);
>   
> -	if (unlikely(!ns || ns->ctrl->state != NVME_CTRL_LIVE))
> -		ns = __nvme_find_path(head);
> +	if (likely(ns && ns->ctrl->state == NVME_CTRL_LIVE &&
> +			nvme_ns_ana_state(ns) == NVME_ANA_OPTIMIZED))
> +		return ns;
> +
> +	ns = __nvme_find_path(head, NVME_ANA_OPTIMIZED);
> +	if (!ns)
> +		ns = __nvme_find_path(head, NVME_ANA_NONOPTIMIZED);
> +	/* XXX: try an inaccessible path as last resort per 8.18.3.3 */
>   	return ns;
>   }
>   
> @@ -239,3 +294,147 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
>   	blk_cleanup_queue(head->disk->queue);
>   	put_disk(head->disk);
>   }
> +
> +static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
> +{
> +	void *base = ctrl->ana_log_buf;
> +	struct nvme_ns *ns;
> +	size_t offset;
> +	int error, i;
> +
> +	/*
> +	 * If anagrpid never changes we don't need to process the namespace
> +	 * lists.
> +	 */
> +	if (ctrl->anacap & (1 << 6))
> +		groups_only = true;
> +
> +	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
> +			groups_only ? NVME_ANA_LOG_RGO : 0,
> +			ctrl->ana_log_buf, ctrl->ana_log_size, 0);
> +	if (error) {
> +		dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
> +		return error;
> +	}
> +
> +	offset = sizeof(struct nvme_ana_rsp_hdr);
> +	for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
> +		struct nvme_ana_group_desc *desc = base + offset;
> +		u32 grpid = le32_to_cpu(desc->grpid);
> +		u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
> +		size_t nsid_buf_size = nr_nsids * sizeof(__le32);
> +
> +		if (WARN_ON_ONCE(grpid == 0))
> +			return -EINVAL;
> +		if (WARN_ON_ONCE(grpid > ctrl->anagrpmax))
> +			return -EINVAL;
> +		if (WARN_ON_ONCE(desc->state == 0))
> +			return -EINVAL;
> +		if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
> +			return -EINVAL;
> +
> +		dev_info(ctrl->device, "ANA group %d: %s.\n",
> +				grpid, nvme_ana_state_names[desc->state]);
> +		WRITE_ONCE(ctrl->ana_state[grpid], desc->state);
> +		offset += sizeof(*desc);
> +		if (!nr_nsids)
> +			continue;
> +
> +		if (WARN_ON_ONCE(groups_only))
> +			return -EINVAL;
> +		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
> +			return -EINVAL;
> +
> +		down_write(&ctrl->namespaces_rwsem);
> +		list_for_each_entry(ns, &ctrl->namespaces, list) {
> +			u32 nsid = le32_to_cpu(desc->nsids[n]);
> +
> +			if (ns->head->ns_id != nsid)
> +				continue;
> +			ns->anagrpid = grpid;
> +			if (++n == nr_nsids)
> +				break;
> +		}
> +		up_write(&ctrl->namespaces_rwsem);
> +		WARN_ON_ONCE(n < nr_nsids);
> +
> +		offset += nsid_buf_size;
> +		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
> +			return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
> +static void nvme_ana_work(struct work_struct *work)
> +{
> +	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
> +
> +	nvme_process_ana_log(ctrl, false);
> +	nvme_kick_requeue_lists(ctrl);

Won't it make sense to kick the requeue list only for namespaces that
are involved in ana state change? Won't attempting this on all
the namespaces might result in retry count increment for namespaces
that are inaccessible? those retries are finite by nvme_max_retries...

[PATCH 05/10] nvme: add ANA support

[Christoph Hellwig]

On Thu, Jun 07, 2018@03:49:20PM +0300, Sagi Grimberg wrote:
>> +		 * XXX: We should verify the controller doesn't die on during
>> +		 * the transition.  But that means we per-group timeout from
>> +		 * when we first hit the change state, so this won't be
>> +		 * entirely trivial..
>
> something is wrong with the sentence I think "that means we per-group
> timeout from when..."
>
> Not sure what you are trying to say.

Reading it again it really looks gibberish..  Hopefully we don't
need the comment any more for next time as Hannes is working on
actually implementing the ANATT timer.

>> +static void nvme_ana_work(struct work_struct *work)
>> +{
>> +	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
>> +
>> +	nvme_process_ana_log(ctrl, false);
>> +	nvme_kick_requeue_lists(ctrl);
>
> Won't it make sense to kick the requeue list only for namespaces that
> are involved in ana state change? Won't attempting this on all
> the namespaces might result in retry count increment for namespaces
> that are inaccessible? those retries are finite by nvme_max_retries...

It makes sense, except that we it would take a lot of effort to figure
out what namespace actually changed due to the group indirection.

What we can do more easily is to only kick it off again on live
namespaces, so I'll look into that.

[PATCH 05/10] nvme: add ANA support

[Christoph Hellwig]

On Thu, Jun 07, 2018@03:05:59PM +0200, Christoph Hellwig wrote:
> It makes sense, except that we it would take a lot of effort to figure
> out what namespace actually changed due to the group indirection.
> 
> What we can do more easily is to only kick it off again on live
> namespaces, so I'll look into that.

The requeue_work will skip before incrementing the retry counter,
so this is not actually an issue.  It might still be nice to
avoid a whole lot of work for the case where there is no live
path, so maybe we can do something about it.

[PATCH 06/10] nvme: don't set gendisks live that don't have an I/O capable path

[Christoph Hellwig]

Only add the gendisk for a ns_head once we have a patch that can actually
do I/O.  Without that we'll get hangs during partition scanning.  To ensure
we don't have dangling unregisters paths make sure to register any missing
ns_head gendisk once the ANA state changes.

Reported-by: Hannes Reinecke <hare at suse.de>
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/host/core.c      |  2 +-
 drivers/nvme/host/multipath.c | 43 ++++++++++++++++++++++++++++++++-----------
 drivers/nvme/host/nvme.h      |  4 ++--
 3 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 67bd73a98a1b..e62de51209b2 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -3081,7 +3081,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 		pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
 			ns->disk->disk_name);
 
-	nvme_mpath_add_disk(ns->head);
+	nvme_mpath_add_disk(ns);
 	nvme_fault_inject_init(ns);
 	return;
  out_unlink_ns:
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 2ab55ca15cec..d30229a21c97 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -264,20 +264,31 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 	return -ENOMEM;
 }
 
-void nvme_mpath_add_disk(struct nvme_ns_head *head)
+static void __nvme_mpath_add_disk(struct nvme_ns *ns)
 {
-	if (!head->disk)
+
+	struct nvme_ns_head *head = ns->head;
+	enum nvme_ana_state state = nvme_ns_ana_state(ns);
+
+	lockdep_assert_held(&head->subsys->lock);
+
+	if (!head->disk ||
+	    (head->disk->flags & GENHD_FL_UP) ||
+	    (state != NVME_ANA_OPTIMIZED && state != NVME_ANA_NONOPTIMIZED))
 		return;
 
-	mutex_lock(&head->subsys->lock);
-	if (!(head->disk->flags & GENHD_FL_UP)) {
-		device_add_disk(&head->subsys->dev, head->disk);
-		if (sysfs_create_group(&disk_to_dev(head->disk)->kobj,
-				&nvme_ns_id_attr_group))
-			pr_warn("%s: failed to create sysfs group for identification\n",
-				head->disk->disk_name);
-	}
-	mutex_unlock(&head->subsys->lock);
+	device_add_disk(&head->subsys->dev, head->disk);
+	if (sysfs_create_group(&disk_to_dev(head->disk)->kobj,
+			&nvme_ns_id_attr_group))
+		pr_warn("%s: failed to create sysfs group for identification\n",
+			head->disk->disk_name);
+}
+
+void nvme_mpath_add_disk(struct nvme_ns *ns)
+{
+	mutex_lock(&ns->head->subsys->lock);
+	__nvme_mpath_add_disk(ns);
+	mutex_unlock(&ns->head->subsys->lock);
 }
 
 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
@@ -363,6 +374,16 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 			return -EINVAL;
 	}
 
+	/*
+	 * Make sure any namespaces that were first discovered in a state where
+	 * we could not do I/O are added once they can perform I/O.
+	 */
+	mutex_lock(&ctrl->subsys->lock);
+	down_write(&ctrl->namespaces_rwsem);
+	list_for_each_entry(ns, &ctrl->namespaces, list)
+		__nvme_mpath_add_disk(ns);
+	up_write(&ctrl->namespaces_rwsem);
+	mutex_unlock(&ctrl->subsys->lock);
 	return 0;
 }
 
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 82a58bd2bf8e..be2585576bad 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -465,7 +465,7 @@ void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
 void nvme_failover_req(struct request *req);
 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
-void nvme_mpath_add_disk(struct nvme_ns_head *head);
+void nvme_mpath_add_disk(struct nvme_ns *ns);
 void nvme_mpath_remove_disk(struct nvme_ns_head *head);
 int nvme_configure_ana(struct nvme_ctrl *ctrl);
 void nvme_deconfigure_ana(struct nvme_ctrl *ctrl);
@@ -512,7 +512,7 @@ static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,
 {
 	return 0;
 }
-static inline void nvme_mpath_add_disk(struct nvme_ns_head *head)
+static inline void nvme_mpath_add_disk(struct nvme_ns *ns)
 {
 }
 static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
-- 
2.14.2

[PATCH 07/10] nvmet: track and limit the number of namespaces per subsystem

[Christoph Hellwig]

TP 4004 introduces a new 'Maximum Number of Allocated Namespaces' field
in the Identify controller data to help the host size resources.  Put
an upper limit on the supported namespaces to be able to support this
value as supporting 32-bits worth of namespaces would lead to very
large buffers.  The limit is completely arbitrary at this point.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/target/admin-cmd.c | 1 +
 drivers/nvme/target/core.c      | 8 +++++++-
 drivers/nvme/target/nvmet.h     | 8 ++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 962532842769..6912548eefc3 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -220,6 +220,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	id->maxcmd = cpu_to_le16(NVMET_MAX_CMD);
 
 	id->nn = cpu_to_le32(ctrl->subsys->max_nsid);
+	id->mnan = cpu_to_le32(NVMET_MAX_NAMESPACES);
 	id->oncs = cpu_to_le16(NVME_CTRL_ONCS_DSM |
 			NVME_CTRL_ONCS_WRITE_ZEROES);
 
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index a03da764ecae..f691ae47c83a 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -332,9 +332,13 @@ static void nvmet_ns_dev_disable(struct nvmet_ns *ns)
 int nvmet_ns_enable(struct nvmet_ns *ns)
 {
 	struct nvmet_subsys *subsys = ns->subsys;
-	int ret = 0;
+	int ret;
 
 	mutex_lock(&subsys->lock);
+	ret = -EMFILE;
+	if (subsys->nr_namespaces == NVMET_MAX_NAMESPACES)
+		goto out_unlock;
+	ret = 0;
 	if (ns->enabled)
 		goto out_unlock;
 
@@ -369,6 +373,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
 
 		list_add_tail_rcu(&ns->dev_link, &old->dev_link);
 	}
+	subsys->nr_namespaces++;
 
 	nvmet_ns_changed(subsys, ns->nsid);
 	ns->enabled = true;
@@ -409,6 +414,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
 	percpu_ref_exit(&ns->ref);
 
 	mutex_lock(&subsys->lock);
+	subsys->nr_namespaces--;
 	nvmet_ns_changed(subsys, ns->nsid);
 	nvmet_ns_dev_disable(ns);
 out_unlock:
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 480dfe10fad9..bd24e2b7317d 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -166,6 +166,7 @@ struct nvmet_subsys {
 	struct kref		ref;
 
 	struct list_head	namespaces;
+	unsigned int		nr_namespaces;
 	unsigned int		max_nsid;
 
 	struct list_head	ctrls;
@@ -357,6 +358,13 @@ u32 nvmet_get_log_page_len(struct nvme_command *cmd);
 #define NVMET_QUEUE_SIZE	1024
 #define NVMET_NR_QUEUES		128
 #define NVMET_MAX_CMD		NVMET_QUEUE_SIZE
+
+/*
+ * Nice round number that makes a list of nsids fit into a page.
+ * Should become tunable at some point in the future.
+ */
+#define NVMET_MAX_NAMESPACES	1024
+
 #define NVMET_KAS		10
 #define NVMET_DISC_KATO		120
 
-- 
2.14.2

[PATCH 07/10] nvmet: track and limit the number of namespaces per subsystem

[Johannes Thumshirn]

Looks good,
Reviewed-by: Johannes Thumshirn <jthumshirn at suse.de>
-- 
Johannes Thumshirn                                          Storage
jthumshirn at suse.de                                +49 911 74053 689
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: Felix Imend?rffer, Jane Smithard, Graham Norton
HRB 21284 (AG N?rnberg)
Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850

[PATCH 07/10] nvmet: track and limit the number of namespaces per subsystem

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:08 +0200
Christoph Hellwig <hch@lst.de> wrote:

> TP 4004 introduces a new 'Maximum Number of Allocated Namespaces'
> field in the Identify controller data to help the host size
> resources.  Put an upper limit on the supported namespaces to be able
> to support this value as supporting 32-bits worth of namespaces would
> lead to very large buffers.  The limit is completely arbitrary at
> this point.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  drivers/nvme/target/admin-cmd.c | 1 +
>  drivers/nvme/target/core.c      | 8 +++++++-
>  drivers/nvme/target/nvmet.h     | 8 ++++++++
>  3 files changed, 16 insertions(+), 1 deletion(-)
> 
Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 08/10] nvmet: add minimal ANA support

[Christoph Hellwig]

Add support for Asynchronous Namespace Access as specified in NVMe 1.3
TP 4004.

Just add a default ANA group 1 that is optimized on all ports.  This is
(and will remain) the default assignment for any namespace not epxlicitly
assigned to another ANA group.  The ANA state can be manually changed
through the configfs interface, including the change state.

Includes fixes and improvements from Hannes Reinecke.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/target/admin-cmd.c | 87 +++++++++++++++++++++++++++++++++++++++--
 drivers/nvme/target/configfs.c  | 10 +++++
 drivers/nvme/target/core.c      | 33 ++++++++++++++++
 drivers/nvme/target/nvmet.h     | 15 +++++++
 4 files changed, 141 insertions(+), 4 deletions(-)

diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 6912548eefc3..c6ae6d3b32f3 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -150,6 +150,69 @@ static void nvmet_execute_get_log_changed_ns(struct nvmet_req *req)
 	nvmet_req_complete(req, status);
 }
 
+static u32 nvmet_format_ana_group(struct nvmet_req *req, u32 grpid,
+		struct nvme_ana_group_desc *desc)
+{
+	struct nvmet_ctrl *ctrl = req->sq->ctrl;
+	struct nvmet_ns *ns;
+	u32 count = 0;
+
+	if (!(req->cmd->get_log_page.lsp & NVME_ANA_LOG_RGO)) {
+		rcu_read_lock();
+		list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link)
+			if (ns->anagrpid == grpid)
+				desc->nsids[count++] = cpu_to_le32(ns->nsid);
+		rcu_read_unlock();
+	}
+
+	desc->grpid = cpu_to_le32(grpid);
+	desc->nnsids = cpu_to_le32(count);
+	desc->chgcnt = cpu_to_le64(nvmet_ana_chgcnt);
+	desc->state = req->port->ana_state[grpid];
+	memset(desc->rsvd17, 0, sizeof(desc->rsvd17));
+	return sizeof(struct nvme_ana_group_desc) + count * sizeof(__le32);
+}
+
+static void nvmet_execute_get_log_page_ana(struct nvmet_req *req)
+{
+	struct nvme_ana_rsp_hdr hdr = { 0, };
+	struct nvme_ana_group_desc *desc;
+	size_t offset = sizeof(struct nvme_ana_rsp_hdr); /* start beyond hdr */
+	size_t len;
+	u32 grpid;
+	u16 ngrps = 0;
+	u16 status;
+
+	status = NVME_SC_INTERNAL;
+	desc = kmalloc(sizeof(struct nvme_ana_group_desc) +
+			NVMET_MAX_NAMESPACES * sizeof(__le32), GFP_KERNEL);
+	if (!desc)
+		goto out;
+
+	down_read(&nvmet_ana_sem);
+	for (grpid = 1; grpid <= NVMET_MAX_ANAGRPS; grpid++) {
+		if (!nvmet_ana_group_enabled[grpid])
+			continue;
+		len = nvmet_format_ana_group(req, grpid, desc);
+		status = nvmet_copy_to_sgl(req, offset, desc, len);
+		if (status)
+			break;
+		offset += len;
+		ngrps++;
+	}
+
+	hdr.chgcnt = cpu_to_le64(nvmet_ana_chgcnt);
+	hdr.ngrps = cpu_to_le16(ngrps);
+	up_read(&nvmet_ana_sem);
+
+	kfree(desc);
+
+	/* copy the header last once we know the number of groups */
+	status = nvmet_copy_to_sgl(req, 0, &hdr, sizeof(hdr));
+out:
+	nvmet_req_complete(req, status);
+}
+
 static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 {
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
@@ -181,8 +244,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	 * the safest is to leave it as zeroes.
 	 */
 
-	/* we support multiple ports and multiples hosts: */
-	id->cmic = (1 << 0) | (1 << 1);
+	/* we support multiple ports, multiples hosts and ANA: */
+	id->cmic = (1 << 0) | (1 << 1) | (1 << 3);
 
 	/* no limit on data transfer sizes for now */
 	id->mdts = 0;
@@ -250,6 +313,11 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 
 	id->msdbd = ctrl->ops->msdbd;
 
+	id->anacap = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4);
+	id->anatt = 10; /* random value */
+	id->anagrpmax = cpu_to_le32(NVMET_MAX_ANAGRPS);
+	id->nanagrpid = cpu_to_le32(NVMET_MAX_ANAGRPS);
+
 	/*
 	 * Meh, we don't really support any power state.  Fake up the same
 	 * values that qemu does.
@@ -291,8 +359,15 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req)
 	 * nuse = ncap = nsze isn't always true, but we have no way to find
 	 * that out from the underlying device.
 	 */
-	id->ncap = id->nuse = id->nsze =
-		cpu_to_le64(ns->size >> ns->blksize_shift);
+	id->ncap = id->nsze = cpu_to_le64(ns->size >> ns->blksize_shift);
+	switch (req->port->ana_state[ns->anagrpid]) {
+	case NVME_ANA_INACCESSIBLE:
+	case NVME_ANA_PERSISTENT_LOSS:
+		break;
+	default:
+		id->nuse = id->nsze;
+		break;
+        }
 
 	/*
 	 * We just provide a single LBA format that matches what the
@@ -306,6 +381,7 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req)
 	 * controllers, but also with any other user of the block device.
 	 */
 	id->nmic = (1 << 0);
+	id->anagrpid = cpu_to_le32(ns->anagrpid);
 
 	memcpy(&id->nguid, &ns->nguid, sizeof(uuid_le));
 
@@ -585,6 +661,9 @@ u16 nvmet_parse_admin_cmd(struct nvmet_req *req)
 		case NVME_LOG_CHANGED_NS:
 			req->execute = nvmet_execute_get_log_changed_ns;
 			return 0;
+		case NVME_LOG_ANA:
+			req->execute = nvmet_execute_get_log_page_ana;
+			return 0;
 		}
 		break;
 	case nvme_admin_identify:
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index ad9ff27234b5..7aa5488dc453 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -861,6 +861,7 @@ static void nvmet_port_release(struct config_item *item)
 {
 	struct nvmet_port *port = to_nvmet_port(item);
 
+	kfree(port->ana_state);
 	kfree(port);
 }
 
@@ -896,6 +897,15 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
 	if (!port)
 		return ERR_PTR(-ENOMEM);
 
+	port->ana_state = kcalloc(NVMET_MAX_ANAGRPS + 1,
+			sizeof(*port->ana_state), GFP_KERNEL);
+	if (!port->ana_state) {
+		kfree(port);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	port->ana_state[NVMET_DEFAULT_ANA_GRPID] = NVME_ANA_OPTIMIZED;
+
 	INIT_LIST_HEAD(&port->entry);
 	INIT_LIST_HEAD(&port->subsystems);
 	INIT_LIST_HEAD(&port->referrals);
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index f691ae47c83a..72c1df24346a 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -39,6 +39,10 @@ static DEFINE_IDA(cntlid_ida);
  */
 DECLARE_RWSEM(nvmet_config_sem);
 
+u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1];
+u64 nvmet_ana_chgcnt;
+DECLARE_RWSEM(nvmet_ana_sem);
+
 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
 		const char *subsysnqn);
 
@@ -425,6 +429,10 @@ void nvmet_ns_free(struct nvmet_ns *ns)
 {
 	nvmet_ns_disable(ns);
 
+	down_write(&nvmet_ana_sem);
+	nvmet_ana_group_enabled[ns->anagrpid]--;
+	up_write(&nvmet_ana_sem);
+
 	kfree(ns->device_path);
 	kfree(ns);
 }
@@ -442,6 +450,12 @@ struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid)
 
 	ns->nsid = nsid;
 	ns->subsys = subsys;
+
+	down_write(&nvmet_ana_sem);
+	ns->anagrpid = NVMET_DEFAULT_ANA_GRPID;
+	nvmet_ana_group_enabled[ns->anagrpid]++;
+	up_write(&nvmet_ana_sem);
+
 	uuid_gen(&ns->uuid);
 
 	return ns;
@@ -548,6 +562,20 @@ int nvmet_sq_init(struct nvmet_sq *sq)
 }
 EXPORT_SYMBOL_GPL(nvmet_sq_init);
 
+static inline u16 nvmet_check_ana_state(struct nvmet_port *port,
+		struct nvmet_ns *ns)
+{
+	enum nvme_ana_state state = port->ana_state[ns->anagrpid];
+
+	if (unlikely(state == NVME_ANA_INACCESSIBLE))
+		return NVME_SC_ANA_INACCESSIBLE;
+	if (unlikely(state == NVME_ANA_PERSISTENT_LOSS))
+		return NVME_SC_ANA_PERSISTENT_LOSS;
+	if (unlikely(state == NVME_ANA_CHANGE))
+		return NVME_SC_ANA_TRANSITION;
+	return 0;
+}
+
 static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
 {
 	struct nvme_command *cmd = req->cmd;
@@ -560,6 +588,9 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
 	req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);
 	if (unlikely(!req->ns))
 		return NVME_SC_INVALID_NS | NVME_SC_DNR;
+	ret = nvmet_check_ana_state(req->port, req->ns);
+	if (unlikely(ret))
+		return ret;
 
 	if (req->ns->file)
 		return nvmet_file_parse_io_cmd(req);
@@ -1107,6 +1138,8 @@ static int __init nvmet_init(void)
 {
 	int error;
 
+	nvmet_ana_group_enabled[NVMET_DEFAULT_ANA_GRPID] = 1;
+
 	error = nvmet_init_discovery();
 	if (error)
 		goto out;
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index bd24e2b7317d..d5d08f70eb0f 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -64,6 +64,7 @@ struct nvmet_ns {
 	loff_t			size;
 	u8			nguid[16];
 	uuid_t			uuid;
+	u32			anagrpid;
 
 	bool			enabled;
 	struct nvmet_subsys	*subsys;
@@ -114,6 +115,7 @@ struct nvmet_port {
 	struct list_head		subsystems;
 	struct config_group		referrals_group;
 	struct list_head		referrals;
+	enum nvme_ana_state		*ana_state;
 	void				*priv;
 	bool				enabled;
 };
@@ -365,6 +367,15 @@ u32 nvmet_get_log_page_len(struct nvme_command *cmd);
  */
 #define NVMET_MAX_NAMESPACES	1024
 
+/*
+ * 0 is not a valid ANA group ID, so we start numbering at 1.
+ *
+ * ANA Group 1 exists without manual intervention, has namespaces assigned to it
+ * by default, and is available in an optimized state through all ports.
+ */
+#define NVMET_MAX_ANAGRPS	1
+#define NVMET_DEFAULT_ANA_GRPID	1
+
 #define NVMET_KAS		10
 #define NVMET_DISC_KATO		120
 
@@ -378,6 +389,10 @@ extern struct nvmet_subsys *nvmet_disc_subsys;
 extern u64 nvmet_genctr;
 extern struct rw_semaphore nvmet_config_sem;
 
+extern u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1];
+extern u64 nvmet_ana_chgcnt;
+extern struct rw_semaphore nvmet_ana_sem;
+
 bool nvmet_host_allowed(struct nvmet_req *req, struct nvmet_subsys *subsys,
 		const char *hostnqn);
 
-- 
2.14.2

[PATCH 08/10] nvmet: add minimal ANA support

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:09 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Add support for Asynchronous Namespace Access as specified in NVMe 1.3
> TP 4004.
> 
> Just add a default ANA group 1 that is optimized on all ports.  This
> is (and will remain) the default assignment for any namespace not
> epxlicitly assigned to another ANA group.  The ANA state can be
> manually changed through the configfs interface, including the change
> state.
> 
> Includes fixes and improvements from Hannes Reinecke.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  drivers/nvme/target/admin-cmd.c | 87
> +++++++++++++++++++++++++++++++++++++++--
> drivers/nvme/target/configfs.c  | 10 +++++
> drivers/nvme/target/core.c      | 33 ++++++++++++++++
> drivers/nvme/target/nvmet.h     | 15 +++++++ 4 files changed, 141
> insertions(+), 4 deletions(-)
> 

Reviewed-by: Hannes Reinecke <hare at suse.com>

Cheers,

Hannes

[PATCH 08/10] nvmet: add minimal ANA support

[Sagi Grimberg]

This looks good,

Reviewed-by: Sagi Grimberg <sagi at rimberg.me>

[PATCH 09/10] nvmet: support configuring ANA groups

[Christoph Hellwig]

Allow creating non-default ANA groups (group ID > 1).  Groups are created
either by assigning the group ID to a namespace, or by creating a configfs
group object under a specific port.  All namespaces assigned to a group
that doesn't have a configfs object for a given port are marked as
inaccessible.

Allow changing the ANA state on a per-port basis by creating an
ana_groups directory under each port, and another directory with an
ana_state file in it.  The default ANA group 1 directory is created
automatically for each port.

For all changes in ANA configuration the ANA change AEN is sent.  We only
keep a global changecount instead of additional per-group changecounts to
keep the implementation as simple as possible.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/target/admin-cmd.c |   1 +
 drivers/nvme/target/configfs.c  | 181 +++++++++++++++++++++++++++++++++++++++-
 drivers/nvme/target/core.c      |  24 ++++++
 drivers/nvme/target/nvmet.h     |  30 ++++++-
 4 files changed, 232 insertions(+), 4 deletions(-)

diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index c6ae6d3b32f3..8de264ece826 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -203,6 +203,7 @@ static void nvmet_execute_get_log_page_ana(struct nvmet_req *req)
 
 	hdr.chgcnt = cpu_to_le64(nvmet_ana_chgcnt);
 	hdr.ngrps = cpu_to_le16(ngrps);
+	clear_bit(NVME_AEN_CFG_ANA_CHANGE, &req->sq->ctrl->aen_masked);
 	up_read(&nvmet_ana_sem);
 
 	kfree(desc);
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index 7aa5488dc453..c5a770ac2f34 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -378,6 +378,39 @@ static ssize_t nvmet_ns_device_nguid_store(struct config_item *item,
 
 CONFIGFS_ATTR(nvmet_ns_, device_nguid);
 
+static ssize_t nvmet_ns_ana_grpid_show(struct config_item *item, char *page)
+{
+	return sprintf(page, "%u\n", to_nvmet_ns(item)->anagrpid);
+}
+
+static ssize_t nvmet_ns_ana_grpid_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct nvmet_ns *ns = to_nvmet_ns(item);
+	u32 oldgrpid, newgrpid;
+	int ret;
+
+	ret = kstrtou32(page, 0, &newgrpid);
+	if (ret)
+		return ret;
+
+	if (newgrpid < 1 || newgrpid > NVMET_MAX_ANAGRPS)
+		return -EINVAL;
+
+	down_write(&nvmet_ana_sem);
+	oldgrpid = ns->anagrpid;
+	nvmet_ana_group_enabled[newgrpid]++;
+	ns->anagrpid = newgrpid;
+	nvmet_ana_group_enabled[oldgrpid]--;
+	nvmet_ana_chgcnt++;
+	up_write(&nvmet_ana_sem);
+
+	nvmet_send_ana_event(ns->subsys);
+	return count;
+}
+
+CONFIGFS_ATTR(nvmet_ns_, ana_grpid);
+
 static ssize_t nvmet_ns_enable_show(struct config_item *item, char *page)
 {
 	return sprintf(page, "%d\n", to_nvmet_ns(item)->enabled);
@@ -407,6 +440,7 @@ static struct configfs_attribute *nvmet_ns_attrs[] = {
 	&nvmet_ns_attr_device_path,
 	&nvmet_ns_attr_device_nguid,
 	&nvmet_ns_attr_device_uuid,
+	&nvmet_ns_attr_ana_grpid,
 	&nvmet_ns_attr_enable,
 	NULL,
 };
@@ -854,6 +888,134 @@ static const struct config_item_type nvmet_referrals_type = {
 	.ct_group_ops	= &nvmet_referral_group_ops,
 };
 
+static struct {
+	enum nvme_ana_state	state;
+	const char		*name;
+} nvmet_ana_state_names[] = {
+	{ NVME_ANA_OPTIMIZED,		"optimized" },
+	{ NVME_ANA_NONOPTIMIZED,	"non-optimized" },
+	{ NVME_ANA_INACCESSIBLE,	"inaccessible" },
+	{ NVME_ANA_PERSISTENT_LOSS,	"persistent-loss" },
+	{ NVME_ANA_CHANGE,		"change" },
+};
+
+static ssize_t nvmet_ana_group_ana_state_show(struct config_item *item,
+		char *page)
+{
+	struct nvmet_ana_group *grp = to_ana_group(item);
+	enum nvme_ana_state state = grp->port->ana_state[grp->grpid];
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nvmet_ana_state_names); i++) {
+		if (state != nvmet_ana_state_names[i].state)
+			continue;
+		return sprintf(page, "%s\n", nvmet_ana_state_names[i].name);
+	}
+
+	return sprintf(page, "\n");
+}
+
+static ssize_t nvmet_ana_group_ana_state_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct nvmet_ana_group *grp = to_ana_group(item);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nvmet_ana_state_names); i++) {
+		if (sysfs_streq(page, nvmet_ana_state_names[i].name))
+			goto found;
+	}
+
+	pr_err("Invalid value '%s' for ana_state\n", page);
+	return -EINVAL;
+
+found:
+	down_write(&nvmet_ana_sem);
+	grp->port->ana_state[grp->grpid] = nvmet_ana_state_names[i].state;
+	nvmet_ana_chgcnt++;
+	up_write(&nvmet_ana_sem);
+
+	nvmet_port_send_ana_event(grp->port);
+	return count;
+}
+
+CONFIGFS_ATTR(nvmet_ana_group_, ana_state);
+
+static struct configfs_attribute *nvmet_ana_group_attrs[] = {
+	&nvmet_ana_group_attr_ana_state,
+	NULL,
+};
+
+static void nvmet_ana_group_release(struct config_item *item)
+{
+	struct nvmet_ana_group *grp = to_ana_group(item);
+
+	if (grp == &grp->port->ana_group1)
+		return;
+
+	down_write(&nvmet_ana_sem);
+	grp->port->ana_state[grp->grpid] = NVME_ANA_INACCESSIBLE;
+	nvmet_ana_group_enabled[grp->grpid]--;
+	up_write(&nvmet_ana_sem);
+
+	nvmet_port_send_ana_event(grp->port);
+	kfree(grp);
+}
+
+static struct configfs_item_operations nvmet_ana_group_item_ops = {
+	.release		= nvmet_ana_group_release,
+};
+
+static const struct config_item_type nvmet_ana_group_type = {
+	.ct_item_ops		= &nvmet_ana_group_item_ops,
+	.ct_attrs		= nvmet_ana_group_attrs,
+	.ct_owner		= THIS_MODULE,
+};
+
+static struct config_group *nvmet_ana_groups_make_group(
+		struct config_group *group, const char *name)
+{
+	struct nvmet_port *port = ana_groups_to_port(&group->cg_item);
+	struct nvmet_ana_group *grp;
+	u32 grpid;
+	int ret;
+
+	ret = kstrtou32(name, 0, &grpid);
+	if (ret)
+		goto out;
+
+	ret = -EINVAL;
+	if (grpid <= 1 || grpid > NVMET_MAX_ANAGRPS)
+		goto out;
+
+	ret = -ENOMEM;
+	grp = kzalloc(sizeof(*grp), GFP_KERNEL);
+	if (!grp)
+		goto out;
+	grp->port = port;
+	grp->grpid = grpid;
+
+	down_write(&nvmet_ana_sem);
+	nvmet_ana_group_enabled[grpid]++;
+	up_write(&nvmet_ana_sem);
+
+	nvmet_port_send_ana_event(grp->port);
+
+	config_group_init_type_name(&grp->group, name, &nvmet_ana_group_type);
+	return &grp->group;
+out:
+	return ERR_PTR(ret);
+}
+
+static struct configfs_group_operations nvmet_ana_groups_group_ops = {
+	.make_group		= nvmet_ana_groups_make_group,
+};
+
+static const struct config_item_type nvmet_ana_groups_type = {
+	.ct_group_ops		= &nvmet_ana_groups_group_ops,
+	.ct_owner		= THIS_MODULE,
+};
+
 /*
  * Ports definitions.
  */
@@ -889,6 +1051,7 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
 {
 	struct nvmet_port *port;
 	u16 portid;
+	u32 i;
 
 	if (kstrtou16(name, 0, &portid))
 		return ERR_PTR(-EINVAL);
@@ -904,7 +1067,12 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	port->ana_state[NVMET_DEFAULT_ANA_GRPID] = NVME_ANA_OPTIMIZED;
+	for (i = 1; i <= NVMET_MAX_ANAGRPS; i++) {
+		if (i == NVMET_DEFAULT_ANA_GRPID)
+			port->ana_state[1] = NVME_ANA_OPTIMIZED;
+		else
+			port->ana_state[i] = NVME_ANA_INACCESSIBLE;
+	}
 
 	INIT_LIST_HEAD(&port->entry);
 	INIT_LIST_HEAD(&port->subsystems);
@@ -921,6 +1089,17 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
 			"referrals", &nvmet_referrals_type);
 	configfs_add_default_group(&port->referrals_group, &port->group);
 
+	config_group_init_type_name(&port->ana_groups_group,
+			"ana_groups", &nvmet_ana_groups_type);
+	configfs_add_default_group(&port->ana_groups_group, &port->group);
+
+	port->ana_group1.port = port;
+	port->ana_group1.grpid = 1;
+	config_group_init_type_name(&port->ana_group1.group,
+			"1", &nvmet_ana_group_type);
+	configfs_add_default_group(&port->ana_group1.group,
+			&port->ana_groups_group);
+
 	return &port->group;
 }
 
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 72c1df24346a..72c573c0a8df 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -193,6 +193,30 @@ static void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid)
 	}
 }
 
+void nvmet_send_ana_event(struct nvmet_subsys *subsys)
+{
+	struct nvmet_ctrl *ctrl;
+
+	mutex_lock(&subsys->lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		if (nvmet_aen_disabled(ctrl, NVME_AEN_CFG_ANA_CHANGE))
+			continue;
+		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
+				NVME_AER_NOTICE_ANA, NVME_LOG_ANA);
+	}
+	mutex_unlock(&subsys->lock);
+}
+
+void nvmet_port_send_ana_event(struct nvmet_port *port)
+{
+	struct nvmet_subsys_link *p;
+
+	down_read(&nvmet_config_sem);
+	list_for_each_entry(p, &port->subsystems, entry)
+		nvmet_send_ana_event(p->subsys);
+	up_read(&nvmet_config_sem);
+}
+
 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops)
 {
 	int ret = 0;
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index d5d08f70eb0f..1f562732baf3 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -24,18 +24,18 @@
 #include <linux/uuid.h>
 #include <linux/nvme.h>
 #include <linux/configfs.h>
+#include <linux/refcount.h>
 #include <linux/rcupdate.h>
 #include <linux/blkdev.h>
 
 #define NVMET_ASYNC_EVENTS		4
 #define NVMET_ERROR_LOG_SLOTS		128
 
-
 /*
  * Supported optional AENs:
  */
 #define NVMET_AEN_CFG_OPTIONAL \
-	NVME_AEN_CFG_NS_ATTR
+	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_ANA_CHANGE)
 
 /*
  * Plus mandatory SMART AENs (we'll never send them, but allow enabling them):
@@ -98,6 +98,18 @@ struct nvmet_sq {
 	struct completion	confirm_done;
 };
 
+struct nvmet_ana_group {
+	struct config_group	group;
+	struct nvmet_port	*port;
+	u32			grpid;
+};
+
+static inline struct nvmet_ana_group *to_ana_group(struct config_item *item)
+{
+	return container_of(to_config_group(item), struct nvmet_ana_group,
+			group);
+}
+
 /**
  * struct nvmet_port -	Common structure to keep port
  *				information for the target.
@@ -115,6 +127,8 @@ struct nvmet_port {
 	struct list_head		subsystems;
 	struct config_group		referrals_group;
 	struct list_head		referrals;
+	struct config_group		ana_groups_group;
+	struct nvmet_ana_group		ana_group1;
 	enum nvme_ana_state		*ana_state;
 	void				*priv;
 	bool				enabled;
@@ -126,6 +140,13 @@ static inline struct nvmet_port *to_nvmet_port(struct config_item *item)
 			group);
 }
 
+static inline struct nvmet_port *ana_groups_to_port(
+		struct config_item *item)
+{
+	return container_of(to_config_group(item), struct nvmet_port,
+			ana_groups_group);
+}
+
 struct nvmet_ctrl {
 	struct nvmet_subsys	*subsys;
 	struct nvmet_cq		**cqs;
@@ -340,6 +361,9 @@ void nvmet_ns_disable(struct nvmet_ns *ns);
 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid);
 void nvmet_ns_free(struct nvmet_ns *ns);
 
+void nvmet_send_ana_event(struct nvmet_subsys *subsys);
+void nvmet_port_send_ana_event(struct nvmet_port *port);
+
 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops);
 void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops);
 
@@ -373,7 +397,7 @@ u32 nvmet_get_log_page_len(struct nvme_command *cmd);
  * ANA Group 1 exists without manual intervention, has namespaces assigned to it
  * by default, and is available in an optimized state through all ports.
  */
-#define NVMET_MAX_ANAGRPS	1
+#define NVMET_MAX_ANAGRPS	128
 #define NVMET_DEFAULT_ANA_GRPID	1
 
 #define NVMET_KAS		10
-- 
2.14.2

[PATCH 09/10] nvmet: support configuring ANA groups

[Hannes Reinecke]

On Wed,  6 Jun 2018 16:33:10 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Allow creating non-default ANA groups (group ID > 1).  Groups are
> created either by assigning the group ID to a namespace, or by
> creating a configfs group object under a specific port.  All
> namespaces assigned to a group that doesn't have a configfs object
> for a given port are marked as inaccessible.
> 
> Allow changing the ANA state on a per-port basis by creating an
> ana_groups directory under each port, and another directory with an
> ana_state file in it.  The default ANA group 1 directory is created
> automatically for each port.
> 
> For all changes in ANA configuration the ANA change AEN is sent.  We
> only keep a global changecount instead of additional per-group
> changecounts to keep the implementation as simple as possible.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  drivers/nvme/target/admin-cmd.c |   1 +
>  drivers/nvme/target/configfs.c  | 181
> +++++++++++++++++++++++++++++++++++++++-
> drivers/nvme/target/core.c      |  24 ++++++
> drivers/nvme/target/nvmet.h     |  30 ++++++- 4 files changed, 232
> insertions(+), 4 deletions(-)
> 
[ .. ]
> @@ -921,6 +1089,17 @@ static struct config_group
> *nvmet_ports_make(struct config_group *group, "referrals",
> &nvmet_referrals_type);
> configfs_add_default_group(&port->referrals_group, &port->group); 
> +	config_group_init_type_name(&port->ana_groups_group,
> +			"ana_groups", &nvmet_ana_groups_type);
> +	configfs_add_default_group(&port->ana_groups_group,
> &port->group); +
> +	port->ana_group1.port = port;
> +	port->ana_group1.grpid = 1;

NVMET_DEFAULT_ANA_GRPID ?

And maybe rename 'ana_group1' to 'ana_default_group' ?

> +	config_group_init_type_name(&port->ana_group1.group,
> +			"1", &nvmet_ana_group_type);

snprintf(grpname, "%d", NVMET_DEFAULT_ANA_GRPID); ?

> +	configfs_add_default_group(&port->ana_group1.group,
> +			&port->ana_groups_group);
> +
>  	return &port->group;
>  }
>  
> diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
> index 72c1df24346a..72c573c0a8df 100644
> --- a/drivers/nvme/target/core.c
> +++ b/drivers/nvme/target/core.c
> @@ -193,6 +193,30 @@ static void nvmet_ns_changed(struct nvmet_subsys
> *subsys, u32 nsid) }
>  }
>  
> +void nvmet_send_ana_event(struct nvmet_subsys *subsys)
> +{
> +	struct nvmet_ctrl *ctrl;
> +
> +	mutex_lock(&subsys->lock);
> +	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
> +		if (nvmet_aen_disabled(ctrl,
> NVME_AEN_CFG_ANA_CHANGE))
> +			continue;
> +		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
> +				NVME_AER_NOTICE_ANA, NVME_LOG_ANA);
> +	}
> +	mutex_unlock(&subsys->lock);
> +}
> +

Isn't this sending too many AENs?
We're changing the ANA state per _port_, which is linked to individual
controllers.
But controllers which are _not_ on that port won't be affected by the
ANA change, so in theory we shouldn't send ANA AENs for not-affected
controllers.

Can't we just add a pointer to the port in the nvmet_ctrl structure and
filter according to the affected ports?

Cheers,

Hannes

[PATCH 09/10] nvmet: support configuring ANA groups

[Christoph Hellwig]

On Thu, Jun 07, 2018@10:09:01AM +0200, Hannes Reinecke wrote:
> > +	configfs_add_default_group(&port->ana_groups_group,
> > &port->group); +
> > +	port->ana_group1.port = port;
> > +	port->ana_group1.grpid = 1;
> 
> NVMET_DEFAULT_ANA_GRPID ?

Yes.

> 
> And maybe rename 'ana_group1' to 'ana_default_group' ?

Ok.

> 
> > +	config_group_init_type_name(&port->ana_group1.group,
> > +			"1", &nvmet_ana_group_type);
> 
> snprintf(grpname, "%d", NVMET_DEFAULT_ANA_GRPID); ?

That'll require a buffer allocation.  But I guess the __stringify
should work here.

> > +void nvmet_send_ana_event(struct nvmet_subsys *subsys)
> > +{
> > +	struct nvmet_ctrl *ctrl;
> > +
> > +	mutex_lock(&subsys->lock);
> > +	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
> > +		if (nvmet_aen_disabled(ctrl,
> > NVME_AEN_CFG_ANA_CHANGE))
> > +			continue;
> > +		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
> > +				NVME_AER_NOTICE_ANA, NVME_LOG_ANA);
> > +	}
> > +	mutex_unlock(&subsys->lock);
> > +}
> > +
> 
> Isn't this sending too many AENs?
> We're changing the ANA state per _port_, which is linked to individual
> controllers.
> But controllers which are _not_ on that port won't be affected by the
> ANA change, so in theory we shouldn't send ANA AENs for not-affected
> controllers.
> 
> Can't we just add a pointer to the port in the nvmet_ctrl structure and
> filter according to the affected ports?

Yes, we should probably filter out only controllers that are on this
port.  I'll take a look at how to best do that.

[PATCH 09/10] nvmet: support configuring ANA groups

[Sagi Grimberg]

On 06/06/2018 05:33 PM, Christoph Hellwig wrote:
> Allow creating non-default ANA groups (group ID > 1).  Groups are created
> either by assigning the group ID to a namespace, or by creating a configfs
> group object under a specific port.  All namespaces assigned to a group
> that doesn't have a configfs object for a given port are marked as
> inaccessible.
> 
> Allow changing the ANA state on a per-port basis by creating an
> ana_groups directory under each port, and another directory with an
> ana_state file in it.  The default ANA group 1 directory is created
> automatically for each port.
> 
> For all changes in ANA configuration the ANA change AEN is sent.  We only
> keep a global changecount instead of additional per-group changecounts to
> keep the implementation as simple as possible.
> 
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>   drivers/nvme/target/admin-cmd.c |   1 +
>   drivers/nvme/target/configfs.c  | 181 +++++++++++++++++++++++++++++++++++++++-
>   drivers/nvme/target/core.c      |  24 ++++++
>   drivers/nvme/target/nvmet.h     |  30 ++++++-
>   4 files changed, 232 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
> index c6ae6d3b32f3..8de264ece826 100644
> --- a/drivers/nvme/target/admin-cmd.c
> +++ b/drivers/nvme/target/admin-cmd.c
> @@ -203,6 +203,7 @@ static void nvmet_execute_get_log_page_ana(struct nvmet_req *req)
>   
>   	hdr.chgcnt = cpu_to_le64(nvmet_ana_chgcnt);
>   	hdr.ngrps = cpu_to_le16(ngrps);
> +	clear_bit(NVME_AEN_CFG_ANA_CHANGE, &req->sq->ctrl->aen_masked);
>   	up_read(&nvmet_ana_sem);
>   
>   	kfree(desc);
> diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
> index 7aa5488dc453..c5a770ac2f34 100644
> --- a/drivers/nvme/target/configfs.c
> +++ b/drivers/nvme/target/configfs.c
> @@ -378,6 +378,39 @@ static ssize_t nvmet_ns_device_nguid_store(struct config_item *item,
>   
>   CONFIGFS_ATTR(nvmet_ns_, device_nguid);
>   
> +static ssize_t nvmet_ns_ana_grpid_show(struct config_item *item, char *page)
> +{
> +	return sprintf(page, "%u\n", to_nvmet_ns(item)->anagrpid);
> +}
> +
> +static ssize_t nvmet_ns_ana_grpid_store(struct config_item *item,
> +		const char *page, size_t count)
> +{
> +	struct nvmet_ns *ns = to_nvmet_ns(item);
> +	u32 oldgrpid, newgrpid;
> +	int ret;
> +
> +	ret = kstrtou32(page, 0, &newgrpid);
> +	if (ret)
> +		return ret;
> +
> +	if (newgrpid < 1 || newgrpid > NVMET_MAX_ANAGRPS)
> +		return -EINVAL;
> +
> +	down_write(&nvmet_ana_sem);
> +	oldgrpid = ns->anagrpid;
> +	nvmet_ana_group_enabled[newgrpid]++;
> +	ns->anagrpid = newgrpid;
> +	nvmet_ana_group_enabled[oldgrpid]--;
> +	nvmet_ana_chgcnt++;
> +	up_write(&nvmet_ana_sem);
> +
> +	nvmet_send_ana_event(ns->subsys);
> +	return count;
> +}
> +
> +CONFIGFS_ATTR(nvmet_ns_, ana_grpid);
> +
>   static ssize_t nvmet_ns_enable_show(struct config_item *item, char *page)
>   {
>   	return sprintf(page, "%d\n", to_nvmet_ns(item)->enabled);
> @@ -407,6 +440,7 @@ static struct configfs_attribute *nvmet_ns_attrs[] = {
>   	&nvmet_ns_attr_device_path,
>   	&nvmet_ns_attr_device_nguid,
>   	&nvmet_ns_attr_device_uuid,
> +	&nvmet_ns_attr_ana_grpid,
>   	&nvmet_ns_attr_enable,
>   	NULL,
>   };
> @@ -854,6 +888,134 @@ static const struct config_item_type nvmet_referrals_type = {
>   	.ct_group_ops	= &nvmet_referral_group_ops,
>   };
>   
> +static struct {
> +	enum nvme_ana_state	state;
> +	const char		*name;
> +} nvmet_ana_state_names[] = {
> +	{ NVME_ANA_OPTIMIZED,		"optimized" },
> +	{ NVME_ANA_NONOPTIMIZED,	"non-optimized" },
> +	{ NVME_ANA_INACCESSIBLE,	"inaccessible" },
> +	{ NVME_ANA_PERSISTENT_LOSS,	"persistent-loss" },
> +	{ NVME_ANA_CHANGE,		"change" },
> +};
> +
> +static ssize_t nvmet_ana_group_ana_state_show(struct config_item *item,
> +		char *page)
> +{
> +	struct nvmet_ana_group *grp = to_ana_group(item);
> +	enum nvme_ana_state state = grp->port->ana_state[grp->grpid];
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(nvmet_ana_state_names); i++) {
> +		if (state != nvmet_ana_state_names[i].state)
> +			continue;
> +		return sprintf(page, "%s\n", nvmet_ana_state_names[i].name);
> +	}
> +
> +	return sprintf(page, "\n");
> +}
> +
> +static ssize_t nvmet_ana_group_ana_state_store(struct config_item *item,
> +		const char *page, size_t count)
> +{
> +	struct nvmet_ana_group *grp = to_ana_group(item);
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(nvmet_ana_state_names); i++) {
> +		if (sysfs_streq(page, nvmet_ana_state_names[i].name))
> +			goto found;
> +	}
> +
> +	pr_err("Invalid value '%s' for ana_state\n", page);
> +	return -EINVAL;
> +
> +found:
> +	down_write(&nvmet_ana_sem);
> +	grp->port->ana_state[grp->grpid] = nvmet_ana_state_names[i].state;
> +	nvmet_ana_chgcnt++;
> +	up_write(&nvmet_ana_sem);
> +
> +	nvmet_port_send_ana_event(grp->port);
> +	return count;
> +}
> +
> +CONFIGFS_ATTR(nvmet_ana_group_, ana_state);
> +
> +static struct configfs_attribute *nvmet_ana_group_attrs[] = {
> +	&nvmet_ana_group_attr_ana_state,
> +	NULL,
> +};
> +
> +static void nvmet_ana_group_release(struct config_item *item)
> +{
> +	struct nvmet_ana_group *grp = to_ana_group(item);
> +
> +	if (grp == &grp->port->ana_group1)
> +		return;
> +
> +	down_write(&nvmet_ana_sem);
> +	grp->port->ana_state[grp->grpid] = NVME_ANA_INACCESSIBLE;
> +	nvmet_ana_group_enabled[grp->grpid]--;
> +	up_write(&nvmet_ana_sem);
> +
> +	nvmet_port_send_ana_event(grp->port);
> +	kfree(grp);
> +}
> +
> +static struct configfs_item_operations nvmet_ana_group_item_ops = {
> +	.release		= nvmet_ana_group_release,
> +};
> +
> +static const struct config_item_type nvmet_ana_group_type = {
> +	.ct_item_ops		= &nvmet_ana_group_item_ops,
> +	.ct_attrs		= nvmet_ana_group_attrs,
> +	.ct_owner		= THIS_MODULE,
> +};
> +
> +static struct config_group *nvmet_ana_groups_make_group(
> +		struct config_group *group, const char *name)
> +{
> +	struct nvmet_port *port = ana_groups_to_port(&group->cg_item);
> +	struct nvmet_ana_group *grp;
> +	u32 grpid;
> +	int ret;
> +
> +	ret = kstrtou32(name, 0, &grpid);
> +	if (ret)
> +		goto out;
> +
> +	ret = -EINVAL;
> +	if (grpid <= 1 || grpid > NVMET_MAX_ANAGRPS)
> +		goto out;
> +
> +	ret = -ENOMEM;
> +	grp = kzalloc(sizeof(*grp), GFP_KERNEL);
> +	if (!grp)
> +		goto out;
> +	grp->port = port;
> +	grp->grpid = grpid;
> +
> +	down_write(&nvmet_ana_sem);
> +	nvmet_ana_group_enabled[grpid]++;
> +	up_write(&nvmet_ana_sem);
> +
> +	nvmet_port_send_ana_event(grp->port);
> +
> +	config_group_init_type_name(&grp->group, name, &nvmet_ana_group_type);
> +	return &grp->group;
> +out:
> +	return ERR_PTR(ret);
> +}
> +
> +static struct configfs_group_operations nvmet_ana_groups_group_ops = {
> +	.make_group		= nvmet_ana_groups_make_group,
> +};
> +
> +static const struct config_item_type nvmet_ana_groups_type = {
> +	.ct_group_ops		= &nvmet_ana_groups_group_ops,
> +	.ct_owner		= THIS_MODULE,
> +};
> +
>   /*
>    * Ports definitions.
>    */
> @@ -889,6 +1051,7 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
>   {
>   	struct nvmet_port *port;
>   	u16 portid;
> +	u32 i;
>   
>   	if (kstrtou16(name, 0, &portid))
>   		return ERR_PTR(-EINVAL);
> @@ -904,7 +1067,12 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
>   		return ERR_PTR(-ENOMEM);
>   	}
>   
> -	port->ana_state[NVMET_DEFAULT_ANA_GRPID] = NVME_ANA_OPTIMIZED;
> +	for (i = 1; i <= NVMET_MAX_ANAGRPS; i++) {
> +		if (i == NVMET_DEFAULT_ANA_GRPID)
> +			port->ana_state[1] = NVME_ANA_OPTIMIZED;
> +		else
> +			port->ana_state[i] = NVME_ANA_INACCESSIBLE;
> +	}
>   
>   	INIT_LIST_HEAD(&port->entry);
>   	INIT_LIST_HEAD(&port->subsystems);
> @@ -921,6 +1089,17 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
>   			"referrals", &nvmet_referrals_type);
>   	configfs_add_default_group(&port->referrals_group, &port->group);
>   
> +	config_group_init_type_name(&port->ana_groups_group,
> +			"ana_groups", &nvmet_ana_groups_type);
> +	configfs_add_default_group(&port->ana_groups_group, &port->group);
> +
> +	port->ana_group1.port = port;
> +	port->ana_group1.grpid = 1;
> +	config_group_init_type_name(&port->ana_group1.group,
> +			"1", &nvmet_ana_group_type);
> +	configfs_add_default_group(&port->ana_group1.group,
> +			&port->ana_groups_group);
> +
>   	return &port->group;
>   }
>   
> diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
> index 72c1df24346a..72c573c0a8df 100644
> --- a/drivers/nvme/target/core.c
> +++ b/drivers/nvme/target/core.c
> @@ -193,6 +193,30 @@ static void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid)
>   	}
>   }
>   
> +void nvmet_send_ana_event(struct nvmet_subsys *subsys)
> +{
> +	struct nvmet_ctrl *ctrl;
> +
> +	mutex_lock(&subsys->lock);
> +	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
> +		if (nvmet_aen_disabled(ctrl, NVME_AEN_CFG_ANA_CHANGE))
> +			continue;
> +		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE,
> +				NVME_AER_NOTICE_ANA, NVME_LOG_ANA);
> +	}
> +	mutex_unlock(&subsys->lock);
> +}
> +
> +void nvmet_port_send_ana_event(struct nvmet_port *port)
> +{
> +	struct nvmet_subsys_link *p;
> +
> +	down_read(&nvmet_config_sem);
> +	list_for_each_entry(p, &port->subsystems, entry)
> +		nvmet_send_ana_event(p->subsys);
> +	up_read(&nvmet_config_sem);
> +}

Won't this send multiple AENs if a subsystem is exposed via multiple
ports? (at least in a theoretical case where the host issues a log
page in between).

[PATCH 09/10] nvmet: support configuring ANA groups

[Christoph Hellwig]

On Thu, Jun 07, 2018@03:58:40PM +0300, Sagi Grimberg wrote:
>> +void nvmet_port_send_ana_event(struct nvmet_port *port)
>> +{
>> +	struct nvmet_subsys_link *p;
>> +
>> +	down_read(&nvmet_config_sem);
>> +	list_for_each_entry(p, &port->subsystems, entry)
>> +		nvmet_send_ana_event(p->subsys);
>> +	up_read(&nvmet_config_sem);
>> +}
>
> Won't this send multiple AENs if a subsystem is exposed via multiple
> ports? (at least in a theoretical case where the host issues a log
> page in between).

Well, we want one AEN per controller always.  Except that this is also
going to send it to controllers on different ports as Hannes already
pointed out.  I'm going to fix that for the next version.

[PATCH 10/10] host fold

[Christoph Hellwig]

---
 drivers/nvme/host/multipath.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index d30229a21c97..f88cc4e401c8 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -62,14 +62,16 @@ void nvme_failover_req(struct request *req)
 	 */
 	switch (nvme_req(req)->status & 0x7ff) {
 	case NVME_SC_ANA_TRANSITION:
-		/*
-		 * XXX: We should verify the controller doesn't die on during
-		 * the transition.  But that means we per-group timeout from
-		 * when we first hit the change state, so this won't be
-		 * entirely trivial..
-		 */
-		nvme_update_ana_state(ns, NVME_ANA_CHANGE);
+	{
+		enum nvme_ana_state old;
+
+		old = xchg(&ns->ctrl->ana_state[ns->anagrpid], NVME_ANA_CHANGE);
+		if (old != NVME_ANA_CHANGE) {
+			// XXX start timer
+			;
+		}
 		break;
+	}
 	case NVME_SC_ANA_PERSISTENT_LOSS:
 		nvme_update_ana_state(ns, NVME_ANA_PERSISTENT_LOSS);
 		break;
@@ -141,7 +143,6 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 	ns = __nvme_find_path(head, NVME_ANA_OPTIMIZED);
 	if (!ns)
 		ns = __nvme_find_path(head, NVME_ANA_NONOPTIMIZED);
-	/* XXX: try an inaccessible path as last resort per 8.18.3.3 */
 	return ns;
 }
 
-- 
2.14.2

[PATCH 10/10] host fold

[Christoph Hellwig]

Ignore this one.  This is just a small part of the not really
working transition timeout tracking.

[PATCH 10/10] host fold

[Hannes Reinecke]

On Wed, 6 Jun 2018 16:46:50 +0200
Christoph Hellwig <hch@lst.de> wrote:

> Ignore this one.  This is just a small part of the not really
> working transition timeout tracking.

I do have an implementation running, will be posting it shortly.
(After rebasing to your latest patchset, that is.)

Got stuck debugging block-layer issues for which I've just posted
patches, otherwise would've done so already.

Cheers,

Hannes

draft ANA support v3

[Sagi Grimberg]

> The big TODO list item for the kernel code is support for transition
> timeouts.
> 
> Besides that we still need nvme-cli and nvmetcli support as well as
> an automated test suite.
> 
> A git tree is available at:
> 
>      git://git.infradead.org/users/hch/block.git nvme-ana
> 
> Gitweb:
> 
>      http://git.infradead.org/users/hch/block.git/shortlog/refs/heads/nvme-ana
> 
> Changes since v2:
>   - support changing the state for the default ANA group
>   - drop ANATT configuration again
>   - delay gendisk registration until we have an I/O capable path
>   - remove nvme_req_needs_failover and the SCT=3 handling in it,
>     we actually do the right thing without it
>   - refactor the get log page helpers

Wish you had accumulated review tags for this one...

draft ANA support v3

[Christoph Hellwig]

On Thu, Jun 07, 2018@03:34:30PM +0300, Sagi Grimberg wrote:
>> Changes since v2:
>>   - support changing the state for the default ANA group
>>   - drop ANATT configuration again
>>   - delay gendisk registration until we have an I/O capable path
>>   - remove nvme_req_needs_failover and the SCT=3 handling in it,
>>     we actually do the right thing without it
>>   - refactor the get log page helpers
>
> Wish you had accumulated review tags for this one...

I've picked them up now.  But the patches are still undergoing
non-trivial changes, so I will probably drop them at least for
some patches again.