Linux-NVME Archive on lore.kernel.org
 help / color / Atom feed
From: jmoyer@redhat.com (Jeff Moyer)
Subject: [PATCH 06/47] blk-mq: add a flags parameter to blk_mq_alloc_request
Date: Tue, 24 Nov 2015 10:19:54 -0500
Message-ID: <x49io4r2zmt.fsf@segfault.boston.devel.redhat.com> (raw)
In-Reply-To: <1448037342-18384-7-git-send-email-hch@lst.de> (Christoph Hellwig's message of "Fri, 20 Nov 2015 17:35:01 +0100")

Christoph Hellwig <hch at lst.de> writes:

> We already have the reserved flag, and a nowait flag awkwardly encoded as
> a gfp_t.  Add a real flags argument to make the scheme more extensible and
> allow for a nicer calling convention.
>
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  block/blk-core.c                  |   11 +-
>  block/blk-mq-tag.c                |   11 +-
>  block/blk-mq.c                    |   20 +-
>  block/blk-mq.h                    |   11 +-
>  block/blk.h                       |    2 +-
>  drivers/block/mtip32xx/mtip32xx.c |    2 +-
>  drivers/nvme/host/core.c          | 1172 +++++++++++++++++++++++++++++++++++++

Christoph, I think you included a bit too much in this patch!  ;-)

-Jeff

>  drivers/nvme/host/pci.c           |   11 +-
>  include/linux/blk-mq.h            |    8 +-
>  9 files changed, 1210 insertions(+), 38 deletions(-)
>  create mode 100644 drivers/nvme/host/core.c
>
> diff --git a/block/blk-core.c b/block/blk-core.c
> index af9c315..d2100aa 100644
> --- a/block/blk-core.c
> +++ b/block/blk-core.c
> @@ -630,7 +630,7 @@ struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
>  }
>  EXPORT_SYMBOL(blk_alloc_queue);
>  
> -int blk_queue_enter(struct request_queue *q, gfp_t gfp)
> +int blk_queue_enter(struct request_queue *q, bool nowait)
>  {
>  	while (true) {
>  		int ret;
> @@ -638,7 +638,7 @@ int blk_queue_enter(struct request_queue *q, gfp_t gfp)
>  		if (percpu_ref_tryget_live(&q->q_usage_counter))
>  			return 0;
>  
> -		if (!gfpflags_allow_blocking(gfp))
> +		if (nowait)
>  			return -EBUSY;
>  
>  		ret = wait_event_interruptible(q->mq_freeze_wq,
> @@ -1284,7 +1284,9 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw,
>  struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
>  {
>  	if (q->mq_ops)
> -		return blk_mq_alloc_request(q, rw, gfp_mask, false);
> +		return blk_mq_alloc_request(q, rw,
> +			(gfp_mask & __GFP_DIRECT_RECLAIM) ?
> +				0 : BLK_MQ_REQ_NOWAIT);
>  	else
>  		return blk_old_get_request(q, rw, gfp_mask);
>  }
> @@ -2052,8 +2054,7 @@ blk_qc_t generic_make_request(struct bio *bio)
>  	do {
>  		struct request_queue *q = bdev_get_queue(bio->bi_bdev);
>  
> -		if (likely(blk_queue_enter(q, __GFP_DIRECT_RECLAIM) == 0)) {
> -
> +		if (likely(blk_queue_enter(q, false) == 0)) {
>  			ret = q->make_request_fn(q, bio);
>  
>  			blk_queue_exit(q);
> diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
> index a07ca34..abdbb47 100644
> --- a/block/blk-mq-tag.c
> +++ b/block/blk-mq-tag.c
> @@ -268,7 +268,7 @@ static int bt_get(struct blk_mq_alloc_data *data,
>  	if (tag != -1)
>  		return tag;
>  
> -	if (!gfpflags_allow_blocking(data->gfp))
> +	if (data->flags & BLK_MQ_REQ_NOWAIT)
>  		return -1;
>  
>  	bs = bt_wait_ptr(bt, hctx);
> @@ -303,7 +303,7 @@ static int bt_get(struct blk_mq_alloc_data *data,
>  		data->ctx = blk_mq_get_ctx(data->q);
>  		data->hctx = data->q->mq_ops->map_queue(data->q,
>  				data->ctx->cpu);
> -		if (data->reserved) {
> +		if (data->flags & BLK_MQ_REQ_RESERVED) {
>  			bt = &data->hctx->tags->breserved_tags;
>  		} else {
>  			last_tag = &data->ctx->last_tag;
> @@ -349,10 +349,9 @@ static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
>  
>  unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
>  {
> -	if (!data->reserved)
> -		return __blk_mq_get_tag(data);
> -
> -	return __blk_mq_get_reserved_tag(data);
> +	if (data->flags & BLK_MQ_REQ_RESERVED)
> +		return __blk_mq_get_reserved_tag(data);
> +	return __blk_mq_get_tag(data);
>  }
>  
>  static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt)
> diff --git a/block/blk-mq.c b/block/blk-mq.c
> index c932605..6da03f1 100644
> --- a/block/blk-mq.c
> +++ b/block/blk-mq.c
> @@ -230,8 +230,8 @@ __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
>  	return NULL;
>  }
>  
> -struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
> -		bool reserved)
> +struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
> +		unsigned int flags)
>  {
>  	struct blk_mq_ctx *ctx;
>  	struct blk_mq_hw_ctx *hctx;
> @@ -239,24 +239,22 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
>  	struct blk_mq_alloc_data alloc_data;
>  	int ret;
>  
> -	ret = blk_queue_enter(q, gfp);
> +	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
>  	if (ret)
>  		return ERR_PTR(ret);
>  
>  	ctx = blk_mq_get_ctx(q);
>  	hctx = q->mq_ops->map_queue(q, ctx->cpu);
> -	blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_DIRECT_RECLAIM,
> -			reserved, ctx, hctx);
> +	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
>  
>  	rq = __blk_mq_alloc_request(&alloc_data, rw);
> -	if (!rq && (gfp & __GFP_DIRECT_RECLAIM)) {
> +	if (!rq && !(flags & BLK_MQ_REQ_NOWAIT)) {
>  		__blk_mq_run_hw_queue(hctx);
>  		blk_mq_put_ctx(ctx);
>  
>  		ctx = blk_mq_get_ctx(q);
>  		hctx = q->mq_ops->map_queue(q, ctx->cpu);
> -		blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
> -				hctx);
> +		blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
>  		rq =  __blk_mq_alloc_request(&alloc_data, rw);
>  		ctx = alloc_data.ctx;
>  	}
> @@ -1181,8 +1179,7 @@ static struct request *blk_mq_map_request(struct request_queue *q,
>  		rw |= REQ_SYNC;
>  
>  	trace_block_getrq(q, bio, rw);
> -	blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
> -			hctx);
> +	blk_mq_set_alloc_data(&alloc_data, q, BLK_MQ_REQ_NOWAIT, ctx, hctx);
>  	rq = __blk_mq_alloc_request(&alloc_data, rw);
>  	if (unlikely(!rq)) {
>  		__blk_mq_run_hw_queue(hctx);
> @@ -1191,8 +1188,7 @@ static struct request *blk_mq_map_request(struct request_queue *q,
>  
>  		ctx = blk_mq_get_ctx(q);
>  		hctx = q->mq_ops->map_queue(q, ctx->cpu);
> -		blk_mq_set_alloc_data(&alloc_data, q,
> -				__GFP_RECLAIM|__GFP_HIGH, false, ctx, hctx);
> +		blk_mq_set_alloc_data(&alloc_data, q, 0, ctx, hctx);
>  		rq = __blk_mq_alloc_request(&alloc_data, rw);
>  		ctx = alloc_data.ctx;
>  		hctx = alloc_data.hctx;
> diff --git a/block/blk-mq.h b/block/blk-mq.h
> index 713820b..eaede8e 100644
> --- a/block/blk-mq.h
> +++ b/block/blk-mq.h
> @@ -96,8 +96,7 @@ static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
>  struct blk_mq_alloc_data {
>  	/* input parameter */
>  	struct request_queue *q;
> -	gfp_t gfp;
> -	bool reserved;
> +	unsigned int flags;
>  
>  	/* input & output parameter */
>  	struct blk_mq_ctx *ctx;
> @@ -105,13 +104,11 @@ struct blk_mq_alloc_data {
>  };
>  
>  static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
> -		struct request_queue *q, gfp_t gfp, bool reserved,
> -		struct blk_mq_ctx *ctx,
> -		struct blk_mq_hw_ctx *hctx)
> +		struct request_queue *q, unsigned int flags,
> +		struct blk_mq_ctx *ctx, struct blk_mq_hw_ctx *hctx)
>  {
>  	data->q = q;
> -	data->gfp = gfp;
> -	data->reserved = reserved;
> +	data->flags = flags;
>  	data->ctx = ctx;
>  	data->hctx = hctx;
>  }
> diff --git a/block/blk.h b/block/blk.h
> index 1d95107..38bf997 100644
> --- a/block/blk.h
> +++ b/block/blk.h
> @@ -72,7 +72,7 @@ void blk_dequeue_request(struct request *rq);
>  void __blk_queue_free_tags(struct request_queue *q);
>  bool __blk_end_bidi_request(struct request *rq, int error,
>  			    unsigned int nr_bytes, unsigned int bidi_bytes);
> -int blk_queue_enter(struct request_queue *q, gfp_t gfp);
> +int blk_queue_enter(struct request_queue *q, bool nowait);
>  void blk_queue_exit(struct request_queue *q);
>  void blk_freeze_queue(struct request_queue *q);
>  
> diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
> index a28a562..cf3b51a 100644
> --- a/drivers/block/mtip32xx/mtip32xx.c
> +++ b/drivers/block/mtip32xx/mtip32xx.c
> @@ -173,7 +173,7 @@ static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd)
>  {
>  	struct request *rq;
>  
> -	rq = blk_mq_alloc_request(dd->queue, 0, __GFP_RECLAIM, true);
> +	rq = blk_mq_alloc_request(dd->queue, 0, BLK_MQ_REQ_RESERVED);
>  	return blk_mq_rq_to_pdu(rq);
>  }
>  
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> new file mode 100644
> index 0000000..53cf507
> --- /dev/null
> +++ b/drivers/nvme/host/core.c
> @@ -0,0 +1,1172 @@
> +/*
> + * NVM Express device driver
> + * Copyright (c) 2011-2014, Intel Corporation.
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + */
> +
> +#include <linux/blkdev.h>
> +#include <linux/blk-mq.h>
> +#include <linux/errno.h>
> +#include <linux/hdreg.h>
> +#include <linux/kernel.h>
> +#include <linux/module.h>
> +#include <linux/list_sort.h>
> +#include <linux/slab.h>
> +#include <linux/types.h>
> +#include <linux/pr.h>
> +#include <linux/ptrace.h>
> +#include <linux/nvme_ioctl.h>
> +#include <linux/t10-pi.h>
> +#include <scsi/sg.h>
> +#include <asm/unaligned.h>
> +
> +#include "nvme.h"
> +
> +#define NVME_MINORS		(1U << MINORBITS)
> +
> +static int nvme_major;
> +module_param(nvme_major, int, 0);
> +
> +static int nvme_char_major;
> +module_param(nvme_char_major, int, 0);
> +
> +static LIST_HEAD(nvme_ctrl_list);
> +DEFINE_SPINLOCK(dev_list_lock);
> +
> +static struct class *nvme_class;
> +
> +static void nvme_free_ns(struct kref *kref)
> +{
> +	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
> +
> +	if (ns->type == NVME_NS_LIGHTNVM)
> +		nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
> +
> +	spin_lock(&dev_list_lock);
> +	ns->disk->private_data = NULL;
> +	spin_unlock(&dev_list_lock);
> +
> +	nvme_put_ctrl(ns->ctrl);
> +	put_disk(ns->disk);
> +	kfree(ns);
> +}
> +
> +static void nvme_put_ns(struct nvme_ns *ns)
> +{
> +	kref_put(&ns->kref, nvme_free_ns);
> +}
> +
> +static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
> +{
> +	struct nvme_ns *ns;
> +
> +	spin_lock(&dev_list_lock);
> +	ns = disk->private_data;
> +	if (ns && !kref_get_unless_zero(&ns->kref))
> +		ns = NULL;
> +	spin_unlock(&dev_list_lock);
> +
> +	return ns;
> +}
> +
> +static struct request *nvme_alloc_request(struct request_queue *q,
> +		struct nvme_command *cmd)
> +{
> +	bool write = cmd->common.opcode & 1;
> +	struct request *req;
> +
> +	req = blk_mq_alloc_request(q, write, 0);
> +	if (IS_ERR(req))
> +		return req;
> +
> +	req->cmd_type = REQ_TYPE_DRV_PRIV;
> +	req->cmd_flags |= REQ_FAILFAST_DRIVER;
> +	req->__data_len = 0;
> +	req->__sector = (sector_t) -1;
> +	req->bio = req->biotail = NULL;
> +
> +	req->cmd = (unsigned char *)cmd;
> +	req->cmd_len = sizeof(struct nvme_command);
> +	req->special = (void *)0;
> +
> +	return req;
> +}
> +
> +/*
> + * Returns 0 on success.  If the result is negative, it's a Linux error code;
> + * if the result is positive, it's an NVM Express status code
> + */
> +int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
> +		void *buffer, unsigned bufflen, u32 *result, unsigned timeout)
> +{
> +	struct request *req;
> +	int ret;
> +
> +	req = nvme_alloc_request(q, cmd);
> +	if (IS_ERR(req))
> +		return PTR_ERR(req);
> +
> +	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
> +
> +	if (buffer && bufflen) {
> +		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
> +		if (ret)
> +			goto out;
> +	}
> +
> +	blk_execute_rq(req->q, NULL, req, 0);
> +	if (result)
> +		*result = (u32)(uintptr_t)req->special;
> +	ret = req->errors;
> + out:
> +	blk_mq_free_request(req);
> +	return ret;
> +}
> +
> +int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
> +		void *buffer, unsigned bufflen)
> +{
> +	return __nvme_submit_sync_cmd(q, cmd, buffer, bufflen, NULL, 0);
> +}
> +
> +int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
> +		void __user *ubuffer, unsigned bufflen,
> +		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
> +		u32 *result, unsigned timeout)
> +{
> +	bool write = cmd->common.opcode & 1;
> +	struct nvme_ns *ns = q->queuedata;
> +	struct gendisk *disk = ns ? ns->disk : NULL;
> +	struct request *req;
> +	struct bio *bio = NULL;
> +	void *meta = NULL;
> +	int ret;
> +
> +	req = nvme_alloc_request(q, cmd);
> +	if (IS_ERR(req))
> +		return PTR_ERR(req);
> +
> +	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
> +
> +	if (ubuffer && bufflen) {
> +		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
> +				GFP_KERNEL);
> +		if (ret)
> +			goto out;
> +		bio = req->bio;
> +
> +		if (!disk)
> +			goto submit;
> +		bio->bi_bdev = bdget_disk(disk, 0);
> +		if (!bio->bi_bdev) {
> +			ret = -ENODEV;
> +			goto out_unmap;
> +		}
> +
> +		if (meta_buffer) {
> +			struct bio_integrity_payload *bip;
> +
> +			meta = kmalloc(meta_len, GFP_KERNEL);
> +			if (!meta) {
> +				ret = -ENOMEM;
> +				goto out_unmap;
> +			}
> +
> +			if (write) {
> +				if (copy_from_user(meta, meta_buffer,
> +						meta_len)) {
> +					ret = -EFAULT;
> +					goto out_free_meta;
> +				}
> +			}
> +
> +			bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
> +			if (!bip) {
> +				ret = -ENOMEM;
> +				goto out_free_meta;
> +			}
> +
> +			bip->bip_iter.bi_size = meta_len;
> +			bip->bip_iter.bi_sector = meta_seed;
> +
> +			ret = bio_integrity_add_page(bio, virt_to_page(meta),
> +					meta_len, offset_in_page(meta));
> +			if (ret != meta_len) {
> +				ret = -ENOMEM;
> +				goto out_free_meta;
> +			}
> +		}
> +	}
> + submit:
> +	blk_execute_rq(req->q, disk, req, 0);
> +	ret = req->errors;
> +	if (result)
> +		*result = (u32)(uintptr_t)req->special;
> +	if (meta && !ret && !write) {
> +		if (copy_to_user(meta_buffer, meta, meta_len))
> +			ret = -EFAULT;
> +	}
> + out_free_meta:
> +	kfree(meta);
> + out_unmap:
> +	if (bio) {
> +		if (disk && bio->bi_bdev)
> +			bdput(bio->bi_bdev);
> +		blk_rq_unmap_user(bio);
> +	}
> + out:
> +	blk_mq_free_request(req);
> +	return ret;
> +}
> +
> +int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
> +		void __user *ubuffer, unsigned bufflen, u32 *result,
> +		unsigned timeout)
> +{
> +	return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
> +			result, timeout);
> +}
> +
> +int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
> +{
> +	struct nvme_command c = { };
> +	int error;
> +
> +	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
> +	c.identify.opcode = nvme_admin_identify;
> +	c.identify.cns = cpu_to_le32(1);
> +
> +	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
> +	if (!*id)
> +		return -ENOMEM;
> +
> +	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
> +			sizeof(struct nvme_id_ctrl));
> +	if (error)
> +		kfree(*id);
> +	return error;
> +}
> +
> +static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
> +{
> +	struct nvme_command c = { };
> +
> +	c.identify.opcode = nvme_admin_identify;
> +	c.identify.cns = cpu_to_le32(2);
> +	c.identify.nsid = cpu_to_le32(nsid);
> +	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
> +}
> +
> +int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
> +		struct nvme_id_ns **id)
> +{
> +	struct nvme_command c = { };
> +	int error;
> +
> +	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
> +	c.identify.opcode = nvme_admin_identify,
> +	c.identify.nsid = cpu_to_le32(nsid),
> +
> +	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
> +	if (!*id)
> +		return -ENOMEM;
> +
> +	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
> +			sizeof(struct nvme_id_ns));
> +	if (error)
> +		kfree(*id);
> +	return error;
> +}
> +
> +int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
> +					dma_addr_t dma_addr, u32 *result)
> +{
> +	struct nvme_command c;
> +
> +	memset(&c, 0, sizeof(c));
> +	c.features.opcode = nvme_admin_get_features;
> +	c.features.nsid = cpu_to_le32(nsid);
> +	c.features.prp1 = cpu_to_le64(dma_addr);
> +	c.features.fid = cpu_to_le32(fid);
> +
> +	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0, result, 0);
> +}
> +
> +int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
> +					dma_addr_t dma_addr, u32 *result)
> +{
> +	struct nvme_command c;
> +
> +	memset(&c, 0, sizeof(c));
> +	c.features.opcode = nvme_admin_set_features;
> +	c.features.prp1 = cpu_to_le64(dma_addr);
> +	c.features.fid = cpu_to_le32(fid);
> +	c.features.dword11 = cpu_to_le32(dword11);
> +
> +	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0, result, 0);
> +}
> +
> +int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
> +{
> +	struct nvme_command c = { };
> +	int error;
> +
> +	c.common.opcode = nvme_admin_get_log_page,
> +	c.common.nsid = cpu_to_le32(0xFFFFFFFF),
> +	c.common.cdw10[0] = cpu_to_le32(
> +			(((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
> +			 NVME_LOG_SMART),
> +
> +	*log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
> +	if (!*log)
> +		return -ENOMEM;
> +
> +	error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
> +			sizeof(struct nvme_smart_log));
> +	if (error)
> +		kfree(*log);
> +	return error;
> +}
> +
> +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
> +{
> +	struct nvme_user_io io;
> +	struct nvme_command c;
> +	unsigned length, meta_len;
> +	void __user *metadata;
> +
> +	if (copy_from_user(&io, uio, sizeof(io)))
> +		return -EFAULT;
> +
> +	switch (io.opcode) {
> +	case nvme_cmd_write:
> +	case nvme_cmd_read:
> +	case nvme_cmd_compare:
> +		break;
> +	default:
> +		return -EINVAL;
> +	}
> +
> +	length = (io.nblocks + 1) << ns->lba_shift;
> +	meta_len = (io.nblocks + 1) * ns->ms;
> +	metadata = (void __user *)(uintptr_t)io.metadata;
> +
> +	if (ns->ext) {
> +		length += meta_len;
> +		meta_len = 0;
> +	} else if (meta_len) {
> +		if ((io.metadata & 3) || !io.metadata)
> +			return -EINVAL;
> +	}
> +
> +	memset(&c, 0, sizeof(c));
> +	c.rw.opcode = io.opcode;
> +	c.rw.flags = io.flags;
> +	c.rw.nsid = cpu_to_le32(ns->ns_id);
> +	c.rw.slba = cpu_to_le64(io.slba);
> +	c.rw.length = cpu_to_le16(io.nblocks);
> +	c.rw.control = cpu_to_le16(io.control);
> +	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
> +	c.rw.reftag = cpu_to_le32(io.reftag);
> +	c.rw.apptag = cpu_to_le16(io.apptag);
> +	c.rw.appmask = cpu_to_le16(io.appmask);
> +
> +	return __nvme_submit_user_cmd(ns->queue, &c,
> +			(void __user *)(uintptr_t)io.addr, length,
> +			metadata, meta_len, io.slba, NULL, 0);
> +}
> +
> +static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
> +			struct nvme_passthru_cmd __user *ucmd)
> +{
> +	struct nvme_passthru_cmd cmd;
> +	struct nvme_command c;
> +	unsigned timeout = 0;
> +	int status;
> +
> +	if (!capable(CAP_SYS_ADMIN))
> +		return -EACCES;
> +	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
> +		return -EFAULT;
> +
> +	memset(&c, 0, sizeof(c));
> +	c.common.opcode = cmd.opcode;
> +	c.common.flags = cmd.flags;
> +	c.common.nsid = cpu_to_le32(cmd.nsid);
> +	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
> +	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
> +	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
> +	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
> +	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
> +	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
> +	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
> +	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
> +
> +	if (cmd.timeout_ms)
> +		timeout = msecs_to_jiffies(cmd.timeout_ms);
> +
> +	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
> +			(void __user *)cmd.addr, cmd.data_len,
> +			&cmd.result, timeout);
> +	if (status >= 0) {
> +		if (put_user(cmd.result, &ucmd->result))
> +			return -EFAULT;
> +	}
> +
> +	return status;
> +}
> +
> +static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
> +		unsigned int cmd, unsigned long arg)
> +{
> +	struct nvme_ns *ns = bdev->bd_disk->private_data;
> +
> +	switch (cmd) {
> +	case NVME_IOCTL_ID:
> +		force_successful_syscall_return();
> +		return ns->ns_id;
> +	case NVME_IOCTL_ADMIN_CMD:
> +		return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
> +	case NVME_IOCTL_IO_CMD:
> +		return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
> +	case NVME_IOCTL_SUBMIT_IO:
> +		return nvme_submit_io(ns, (void __user *)arg);
> +	case SG_GET_VERSION_NUM:
> +		return nvme_sg_get_version_num((void __user *)arg);
> +	case SG_IO:
> +		return nvme_sg_io(ns, (void __user *)arg);
> +	default:
> +		return -ENOTTY;
> +	}
> +}
> +
> +#ifdef CONFIG_COMPAT
> +static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
> +			unsigned int cmd, unsigned long arg)
> +{
> +	switch (cmd) {
> +	case SG_IO:
> +		return -ENOIOCTLCMD;
> +	}
> +	return nvme_ioctl(bdev, mode, cmd, arg);
> +}
> +#else
> +#define nvme_compat_ioctl	NULL
> +#endif
> +
> +static int nvme_open(struct block_device *bdev, fmode_t mode)
> +{
> +	return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
> +}
> +
> +static void nvme_release(struct gendisk *disk, fmode_t mode)
> +{
> +	nvme_put_ns(disk->private_data);
> +}
> +
> +static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
> +{
> +	/* some standard values */
> +	geo->heads = 1 << 6;
> +	geo->sectors = 1 << 5;
> +	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
> +	return 0;
> +}
> +
> +#ifdef CONFIG_BLK_DEV_INTEGRITY
> +static void nvme_init_integrity(struct nvme_ns *ns)
> +{
> +	struct blk_integrity integrity;
> +
> +	switch (ns->pi_type) {
> +	case NVME_NS_DPS_PI_TYPE3:
> +		integrity.profile = &t10_pi_type3_crc;
> +		break;
> +	case NVME_NS_DPS_PI_TYPE1:
> +	case NVME_NS_DPS_PI_TYPE2:
> +		integrity.profile = &t10_pi_type1_crc;
> +		break;
> +	default:
> +		integrity.profile = NULL;
> +		break;
> +	}
> +	integrity.tuple_size = ns->ms;
> +	blk_integrity_register(ns->disk, &integrity);
> +	blk_queue_max_integrity_segments(ns->queue, 1);
> +}
> +#else
> +static void nvme_init_integrity(struct nvme_ns *ns)
> +{
> +}
> +#endif /* CONFIG_BLK_DEV_INTEGRITY */
> +
> +static void nvme_config_discard(struct nvme_ns *ns)
> +{
> +	u32 logical_block_size = queue_logical_block_size(ns->queue);
> +	ns->queue->limits.discard_zeroes_data = 0;
> +	ns->queue->limits.discard_alignment = logical_block_size;
> +	ns->queue->limits.discard_granularity = logical_block_size;
> +	blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
> +	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
> +}
> +
> +static int nvme_revalidate_disk(struct gendisk *disk)
> +{
> +	struct nvme_ns *ns = disk->private_data;
> +	struct nvme_id_ns *id;
> +	u8 lbaf, pi_type;
> +	u16 old_ms;
> +	unsigned short bs;
> +
> +	if (nvme_identify_ns(ns->ctrl, ns->ns_id, &id)) {
> +		dev_warn(ns->ctrl->dev, "%s: Identify failure nvme%dn%d\n",
> +				__func__, ns->ctrl->instance, ns->ns_id);
> +		return -ENODEV;
> +	}
> +	if (id->ncap == 0) {
> +		kfree(id);
> +		return -ENODEV;
> +	}
> +
> +	if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
> +		if (nvme_nvm_register(ns->queue, disk->disk_name)) {
> +			dev_warn(ns->ctrl->dev,
> +				"%s: LightNVM init failure\n", __func__);
> +			kfree(id);
> +			return -ENODEV;
> +		}
> +		ns->type = NVME_NS_LIGHTNVM;
> +	}
> +
> +	old_ms = ns->ms;
> +	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
> +	ns->lba_shift = id->lbaf[lbaf].ds;
> +	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
> +	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
> +
> +	/*
> +	 * If identify namespace failed, use default 512 byte block size so
> +	 * block layer can use before failing read/write for 0 capacity.
> +	 */
> +	if (ns->lba_shift == 0)
> +		ns->lba_shift = 9;
> +	bs = 1 << ns->lba_shift;
> +	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
> +	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
> +					id->dps & NVME_NS_DPS_PI_MASK : 0;
> +
> +	blk_mq_freeze_queue(disk->queue);
> +	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
> +				ns->ms != old_ms ||
> +				bs != queue_logical_block_size(disk->queue) ||
> +				(ns->ms && ns->ext)))
> +		blk_integrity_unregister(disk);
> +
> +	ns->pi_type = pi_type;
> +	blk_queue_logical_block_size(ns->queue, bs);
> +
> +	if (ns->ms && !blk_get_integrity(disk) && !ns->ext)
> +		nvme_init_integrity(ns);
> +	if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
> +		set_capacity(disk, 0);
> +	else
> +		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
> +
> +	if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
> +		nvme_config_discard(ns);
> +	blk_mq_unfreeze_queue(disk->queue);
> +
> +	kfree(id);
> +	return 0;
> +}
> +
> +static char nvme_pr_type(enum pr_type type)
> +{
> +	switch (type) {
> +	case PR_WRITE_EXCLUSIVE:
> +		return 1;
> +	case PR_EXCLUSIVE_ACCESS:
> +		return 2;
> +	case PR_WRITE_EXCLUSIVE_REG_ONLY:
> +		return 3;
> +	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
> +		return 4;
> +	case PR_WRITE_EXCLUSIVE_ALL_REGS:
> +		return 5;
> +	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
> +		return 6;
> +	default:
> +		return 0;
> +	}
> +};
> +
> +static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
> +				u64 key, u64 sa_key, u8 op)
> +{
> +	struct nvme_ns *ns = bdev->bd_disk->private_data;
> +	struct nvme_command c;
> +	u8 data[16] = { 0, };
> +
> +	put_unaligned_le64(key, &data[0]);
> +	put_unaligned_le64(sa_key, &data[8]);
> +
> +	memset(&c, 0, sizeof(c));
> +	c.common.opcode = op;
> +	c.common.nsid = cpu_to_le32(ns->ns_id);
> +	c.common.cdw10[0] = cpu_to_le32(cdw10);
> +
> +	return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
> +}
> +
> +static int nvme_pr_register(struct block_device *bdev, u64 old,
> +		u64 new, unsigned flags)
> +{
> +	u32 cdw10;
> +
> +	if (flags & ~PR_FL_IGNORE_KEY)
> +		return -EOPNOTSUPP;
> +
> +	cdw10 = old ? 2 : 0;
> +	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
> +	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
> +	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
> +}
> +
> +static int nvme_pr_reserve(struct block_device *bdev, u64 key,
> +		enum pr_type type, unsigned flags)
> +{
> +	u32 cdw10;
> +
> +	if (flags & ~PR_FL_IGNORE_KEY)
> +		return -EOPNOTSUPP;
> +
> +	cdw10 = nvme_pr_type(type) << 8;
> +	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
> +	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
> +}
> +
> +static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
> +		enum pr_type type, bool abort)
> +{
> +	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
> +	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
> +}
> +
> +static int nvme_pr_clear(struct block_device *bdev, u64 key)
> +{
> +	u32 cdw10 = 1 | key ? 1 << 3 : 0;
> +	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
> +}
> +
> +static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
> +{
> +	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
> +	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
> +}
> +
> +static const struct pr_ops nvme_pr_ops = {
> +	.pr_register	= nvme_pr_register,
> +	.pr_reserve	= nvme_pr_reserve,
> +	.pr_release	= nvme_pr_release,
> +	.pr_preempt	= nvme_pr_preempt,
> +	.pr_clear	= nvme_pr_clear,
> +};
> +
> +static const struct block_device_operations nvme_fops = {
> +	.owner		= THIS_MODULE,
> +	.ioctl		= nvme_ioctl,
> +	.compat_ioctl	= nvme_compat_ioctl,
> +	.open		= nvme_open,
> +	.release	= nvme_release,
> +	.getgeo		= nvme_getgeo,
> +	.revalidate_disk= nvme_revalidate_disk,
> +	.pr_ops		= &nvme_pr_ops,
> +};
> +
> +/*
> + * Initialize the cached copies of the Identify data and various controller
> + * register in our nvme_ctrl structure.  This should be called as soon as
> + * the admin queue is fully up and running.
> + */
> +int nvme_init_identify(struct nvme_ctrl *ctrl)
> +{
> +	struct nvme_id_ctrl *id;
> +	u64 cap;
> +	int ret, page_shift;
> +
> +	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
> +	if (ret) {
> +		dev_err(ctrl->dev, "Reading VS failed (%d)\n", ret);
> +		return ret;
> +	}
> +
> +	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
> +	if (ret) {
> +		dev_err(ctrl->dev, "Reading CAP failed (%d)\n", ret);
> +		return ret;
> +	}
> +	page_shift = NVME_CAP_MPSMIN(cap) + 12;
> +	ctrl->page_size = 1 << page_shift;
> +
> +	if (ctrl->vs >= NVME_VS(1, 1))
> +		ctrl->subsystem = NVME_CAP_NSSRC(cap);
> +
> +	ret = nvme_identify_ctrl(ctrl, &id);
> +	if (ret) {
> +		dev_err(ctrl->dev, "Identify Controller failed (%d)\n", ret);
> +		return -EIO;
> +	}
> +
> +	ctrl->oncs = le16_to_cpup(&id->oncs);
> +	atomic_set(&ctrl->abort_limit, id->acl + 1);
> +	ctrl->vwc = id->vwc;
> +	memcpy(ctrl->serial, id->sn, sizeof(id->sn));
> +	memcpy(ctrl->model, id->mn, sizeof(id->mn));
> +	memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
> +	if (id->mdts)
> +		ctrl->max_hw_sectors = 1 << (id->mdts + page_shift - 9);
> +	else
> +		ctrl->max_hw_sectors = UINT_MAX;
> +
> +	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && id->vs[3]) {
> +		unsigned int max_hw_sectors;
> +
> +		ctrl->stripe_size = 1 << (id->vs[3] + page_shift);
> +		max_hw_sectors = ctrl->stripe_size >> (page_shift - 9);
> +		if (ctrl->max_hw_sectors) {
> +			ctrl->max_hw_sectors = min(max_hw_sectors,
> +							ctrl->max_hw_sectors);
> +		} else {
> +			ctrl->max_hw_sectors = max_hw_sectors;
> +		}
> +	}
> +
> +	kfree(id);
> +	return 0;
> +}
> +
> +static int nvme_dev_open(struct inode *inode, struct file *file)
> +{
> +	struct nvme_ctrl *ctrl;
> +	int instance = iminor(inode);
> +	int ret = -ENODEV;
> +
> +	spin_lock(&dev_list_lock);
> +	list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
> +		if (ctrl->instance != instance)
> +			continue;
> +
> +		if (!ctrl->admin_q) {
> +			ret = -EWOULDBLOCK;
> +			break;
> +		}
> +		if (!kref_get_unless_zero(&ctrl->kref))
> +			break;
> +		file->private_data = ctrl;
> +		ret = 0;
> +		break;
> +	}
> +	spin_unlock(&dev_list_lock);
> +
> +	return ret;
> +}
> +
> +static int nvme_dev_release(struct inode *inode, struct file *file)
> +{
> +	nvme_put_ctrl(file->private_data);
> +	return 0;
> +}
> +
> +static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
> +		unsigned long arg)
> +{
> +	struct nvme_ctrl *ctrl = file->private_data;
> +	void __user *argp = (void __user *)arg;
> +	struct nvme_ns *ns;
> +
> +	switch (cmd) {
> +	case NVME_IOCTL_ADMIN_CMD:
> +		return nvme_user_cmd(ctrl, NULL, argp);
> +	case NVME_IOCTL_IO_CMD:
> +		if (list_empty(&ctrl->namespaces))
> +			return -ENOTTY;
> +		ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
> +		return nvme_user_cmd(ctrl, ns, argp);
> +	case NVME_IOCTL_RESET:
> +		dev_warn(ctrl->dev, "resetting controller\n");
> +		return ctrl->ops->reset_ctrl(ctrl);
> +	case NVME_IOCTL_SUBSYS_RESET:
> +		return nvme_reset_subsystem(ctrl);
> +	default:
> +		return -ENOTTY;
> +	}
> +}
> +
> +static const struct file_operations nvme_dev_fops = {
> +	.owner		= THIS_MODULE,
> +	.open		= nvme_dev_open,
> +	.release	= nvme_dev_release,
> +	.unlocked_ioctl	= nvme_dev_ioctl,
> +	.compat_ioctl	= nvme_dev_ioctl,
> +};
> +
> +static ssize_t nvme_sysfs_reset(struct device *dev,
> +				struct device_attribute *attr, const char *buf,
> +				size_t count)
> +{
> +	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
> +	int ret;
> +
> +	ret = ctrl->ops->reset_ctrl(ctrl);
> +	if (ret < 0)
> +		return ret;
> +	return count;
> +}
> +static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
> +
> +static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
> +{
> +	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
> +	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
> +
> +	return nsa->ns_id - nsb->ns_id;
> +}
> +
> +static struct nvme_ns *nvme_find_ns(struct nvme_ctrl *ctrl, unsigned nsid)
> +{
> +	struct nvme_ns *ns;
> +
> +	list_for_each_entry(ns, &ctrl->namespaces, list) {
> +		if (ns->ns_id == nsid)
> +			return ns;
> +		if (ns->ns_id > nsid)
> +			break;
> +	}
> +	return NULL;
> +}
> +
> +static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
> +{
> +	struct nvme_ns *ns;
> +	struct gendisk *disk;
> +	int node = dev_to_node(ctrl->dev);
> +
> +	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
> +	if (!ns)
> +		return;
> +
> +	ns->queue = blk_mq_init_queue(ctrl->tagset);
> +	if (IS_ERR(ns->queue))
> +		goto out_free_ns;
> +	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
> +	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
> +	ns->queue->queuedata = ns;
> +	ns->ctrl = ctrl;
> +
> +	disk = alloc_disk_node(0, node);
> +	if (!disk)
> +		goto out_free_queue;
> +
> +	kref_init(&ns->kref);
> +	ns->ns_id = nsid;
> +	ns->disk = disk;
> +	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
> +
> +	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
> +	if (ctrl->max_hw_sectors) {
> +		blk_queue_max_hw_sectors(ns->queue, ctrl->max_hw_sectors);
> +		blk_queue_max_segments(ns->queue,
> +			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1);
> +	}
> +	if (ctrl->stripe_size)
> +		blk_queue_chunk_sectors(ns->queue, ctrl->stripe_size >> 9);
> +	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
> +		blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
> +	blk_queue_virt_boundary(ns->queue, ctrl->page_size - 1);
> +
> +	disk->major = nvme_major;
> +	disk->first_minor = 0;
> +	disk->fops = &nvme_fops;
> +	disk->private_data = ns;
> +	disk->queue = ns->queue;
> +	disk->driverfs_dev = ctrl->device;
> +	disk->flags = GENHD_FL_EXT_DEVT;
> +	sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance, nsid);
> +
> +	if (nvme_revalidate_disk(ns->disk))
> +		goto out_free_disk;
> +
> +	list_add_tail(&ns->list, &ctrl->namespaces);
> +	kref_get(&ctrl->kref);
> +	if (ns->type != NVME_NS_LIGHTNVM)
> +		add_disk(ns->disk);
> +
> +	return;
> + out_free_disk:
> +	kfree(disk);
> + out_free_queue:
> +	blk_cleanup_queue(ns->queue);
> + out_free_ns:
> +	kfree(ns);
> +}
> +
> +static void nvme_ns_remove(struct nvme_ns *ns)
> +{
> +	bool kill = nvme_io_incapable(ns->ctrl) &&
> +			!blk_queue_dying(ns->queue);
> +
> +	if (kill)
> +		blk_set_queue_dying(ns->queue);
> +	if (ns->disk->flags & GENHD_FL_UP) {
> +		if (blk_get_integrity(ns->disk))
> +			blk_integrity_unregister(ns->disk);
> +		del_gendisk(ns->disk);
> +	}
> +	if (kill || !blk_queue_dying(ns->queue)) {
> +		blk_mq_abort_requeue_list(ns->queue);
> +		blk_cleanup_queue(ns->queue);
> +	}
> +	list_del_init(&ns->list);
> +	nvme_put_ns(ns);
> +}
> +
> +static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
> +{
> +	struct nvme_ns *ns;
> +
> +	ns = nvme_find_ns(ctrl, nsid);
> +	if (ns) {
> +		if (revalidate_disk(ns->disk))
> +			nvme_ns_remove(ns);
> +	} else
> +		nvme_alloc_ns(ctrl, nsid);
> +}
> +
> +static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
> +{
> +	struct nvme_ns *ns;
> +	__le32 *ns_list;
> +	unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
> +	int ret = 0;
> +
> +	ns_list = kzalloc(0x1000, GFP_KERNEL);
> +	if (!ns_list)
> +		return -ENOMEM;
> +
> +	for (i = 0; i < num_lists; i++) {
> +		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
> +		if (ret)
> +			goto out;
> +
> +		for (j = 0; j < min(nn, 1024U); j++) {
> +			nsid = le32_to_cpu(ns_list[j]);
> +			if (!nsid)
> +				goto out;
> +
> +			nvme_validate_ns(ctrl, nsid);
> +
> +			while (++prev < nsid) {
> +				ns = nvme_find_ns(ctrl, prev);
> +				if (ns)
> +					nvme_ns_remove(ns);
> +			}
> +		}
> +		nn -= j;
> +	}
> + out:
> +	kfree(ns_list);
> +	return ret;
> +}
> +
> +static void __nvme_scan_namespaces(struct nvme_ctrl *ctrl, unsigned nn)
> +{
> +	struct nvme_ns *ns, *next;
> +	unsigned i;
> +
> +	for (i = 1; i <= nn; i++)
> +		nvme_validate_ns(ctrl, i);
> +
> +	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
> +		if (ns->ns_id > nn)
> +			nvme_ns_remove(ns);
> +	}
> +}
> +
> +void nvme_scan_namespaces(struct nvme_ctrl *ctrl)
> +{
> +	struct nvme_id_ctrl *id;
> +	unsigned nn;
> +
> +	if (nvme_identify_ctrl(ctrl, &id))
> +		return;
> +
> +	nn = le32_to_cpu(id->nn);
> +	if (ctrl->vs >= NVME_VS(1, 1)) {
> +		if (!nvme_scan_ns_list(ctrl, nn))
> +			goto done;
> +	}
> +	__nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn));
> + done:
> +	list_sort(NULL, &ctrl->namespaces, ns_cmp);
> +	kfree(id);
> +}
> +
> +void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
> +{
> +	struct nvme_ns *ns, *next;
> +
> +	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
> +		nvme_ns_remove(ns);
> +}
> +
> +static DEFINE_IDA(nvme_instance_ida);
> +
> +static int nvme_set_instance(struct nvme_ctrl *ctrl)
> +{
> +	int instance, error;
> +
> +	do {
> +		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
> +			return -ENODEV;
> +
> +		spin_lock(&dev_list_lock);
> +		error = ida_get_new(&nvme_instance_ida, &instance);
> +		spin_unlock(&dev_list_lock);
> +	} while (error == -EAGAIN);
> +
> +	if (error)
> +		return -ENODEV;
> +
> +	ctrl->instance = instance;
> +	return 0;
> +}
> +
> +static void nvme_release_instance(struct nvme_ctrl *ctrl)
> +{
> +	spin_lock(&dev_list_lock);
> +	ida_remove(&nvme_instance_ida, ctrl->instance);
> +	spin_unlock(&dev_list_lock);
> +}
> +
> +void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
> + {
> +	device_remove_file(ctrl->device, &dev_attr_reset_controller);
> +	device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
> +
> +	spin_lock(&dev_list_lock);
> +	list_del(&ctrl->node);
> +	spin_unlock(&dev_list_lock);
> +}
> +
> +static void nvme_free_ctrl(struct kref *kref)
> +{
> +	struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
> +
> +	put_device(ctrl->device);
> +	nvme_release_instance(ctrl);
> +
> +	ctrl->ops->free_ctrl(ctrl);
> +}
> +
> +void nvme_put_ctrl(struct nvme_ctrl *ctrl)
> +{
> +	kref_put(&ctrl->kref, nvme_free_ctrl);
> +}
> +
> +/*
> + * Initialize a NVMe controller structures.  This needs to be called during
> + * earliest initialization so that we have the initialized structured around
> + * during probing.
> + */
> +int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
> +		const struct nvme_ctrl_ops *ops, u16 vendor,
> +		unsigned long quirks)
> +{
> +	int ret;
> +
> +	INIT_LIST_HEAD(&ctrl->namespaces);
> +	kref_init(&ctrl->kref);
> +	ctrl->dev = dev;
> +	ctrl->ops = ops;
> +	ctrl->vendor = vendor;
> +	ctrl->quirks = quirks;
> +
> +	ret = nvme_set_instance(ctrl);
> +	if (ret)
> +		goto out;
> +
> +	ctrl->device = device_create(nvme_class, ctrl->dev,
> +				MKDEV(nvme_char_major, ctrl->instance),
> +				dev, "nvme%d", ctrl->instance);
> +	if (IS_ERR(ctrl->device)) {
> +		ret = PTR_ERR(ctrl->device);
> +		goto out_release_instance;
> +	}
> +	get_device(ctrl->device);
> +	dev_set_drvdata(ctrl->device, ctrl);
> +
> +	ret = device_create_file(ctrl->device, &dev_attr_reset_controller);
> +	if (ret)
> +		goto out_put_device;
> +
> +	spin_lock(&dev_list_lock);
> +	list_add_tail(&ctrl->node, &nvme_ctrl_list);
> +	spin_unlock(&dev_list_lock);
> +
> +	return 0;
> +
> +out_put_device:
> +	put_device(ctrl->device);
> +	device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
> +out_release_instance:
> +	nvme_release_instance(ctrl);
> +out:
> +	return ret;
> +}
> +
> +int __init nvme_core_init(void)
> +{
> +	int result;
> +
> +	result = register_blkdev(nvme_major, "nvme");
> +	if (result < 0)
> +		return result;
> +	else if (result > 0)
> +		nvme_major = result;
> +
> +	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
> +							&nvme_dev_fops);
> +	if (result < 0)
> +		goto unregister_blkdev;
> +	else if (result > 0)
> +		nvme_char_major = result;
> +
> +	nvme_class = class_create(THIS_MODULE, "nvme");
> +	if (IS_ERR(nvme_class)) {
> +		result = PTR_ERR(nvme_class);
> +		goto unregister_chrdev;
> +	}
> +
> +	return 0;
> +
> + unregister_chrdev:
> +	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
> + unregister_blkdev:
> +	unregister_blkdev(nvme_major, "nvme");
> +	return result;
> +}
> +
> +void nvme_core_exit(void)
> +{
> +	unregister_blkdev(nvme_major, "nvme");
> +	class_destroy(nvme_class);
> +	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
> +}
> diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
> index 9444884..5c5f455 100644
> --- a/drivers/nvme/host/pci.c
> +++ b/drivers/nvme/host/pci.c
> @@ -1040,7 +1040,7 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
>  	struct request *req;
>  	int ret;
>  
> -	req = blk_mq_alloc_request(q, write, GFP_KERNEL, false);
> +	req = blk_mq_alloc_request(q, write, 0);
>  	if (IS_ERR(req))
>  		return PTR_ERR(req);
>  
> @@ -1093,7 +1093,8 @@ static int nvme_submit_async_admin_req(struct nvme_dev *dev)
>  	struct nvme_cmd_info *cmd_info;
>  	struct request *req;
>  
> -	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, true);
> +	req = blk_mq_alloc_request(dev->admin_q, WRITE,
> +			BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED);
>  	if (IS_ERR(req))
>  		return PTR_ERR(req);
>  
> @@ -1118,7 +1119,7 @@ static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
>  	struct request *req;
>  	struct nvme_cmd_info *cmd_rq;
>  
> -	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
> +	req = blk_mq_alloc_request(dev->admin_q, WRITE, 0);
>  	if (IS_ERR(req))
>  		return PTR_ERR(req);
>  
> @@ -1319,8 +1320,8 @@ static void nvme_abort_req(struct request *req)
>  	if (!dev->abort_limit)
>  		return;
>  
> -	abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC,
> -									false);
> +	abort_req = blk_mq_alloc_request(dev->admin_q, WRITE,
> +			BLK_MQ_REQ_NOWAIT);
>  	if (IS_ERR(abort_req))
>  		return;
>  
> diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
> index daf17d7..7fc9296 100644
> --- a/include/linux/blk-mq.h
> +++ b/include/linux/blk-mq.h
> @@ -188,8 +188,14 @@ void blk_mq_insert_request(struct request *, bool, bool, bool);
>  void blk_mq_free_request(struct request *rq);
>  void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *, struct request *rq);
>  bool blk_mq_can_queue(struct blk_mq_hw_ctx *);
> +
> +enum {
> +	BLK_MQ_REQ_NOWAIT	= (1 << 0), /* return when out of requests */
> +	BLK_MQ_REQ_RESERVED	= (1 << 1), /* allocate from reserved pool */
> +};
> +
>  struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
> -		gfp_t gfp, bool reserved);
> +		unsigned int flags);
>  struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag);
>  struct cpumask *blk_mq_tags_cpumask(struct blk_mq_tags *tags);

  reply index

Thread overview: 63+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2015-11-20 16:34 NVMe mega patchbomb for Linux 4.5-rc Christoph Hellwig
2015-11-20 16:34 ` [PATCH 01/47] nvme: add missing unmaps in nvme_queue_rq Christoph Hellwig
2015-11-20 16:34 ` [PATCH 02/47] block: fix blk_abort_request for blk-mq drivers Christoph Hellwig
2015-11-20 21:43   ` Jeff Moyer
2015-11-20 21:47     ` Jens Axboe
2015-11-20 21:54       ` Jeff Moyer
2015-11-20 22:20   ` Jeff Moyer
2015-11-21  7:34     ` Christoph Hellwig
2015-11-20 16:34 ` [PATCH 03/47] block: defer timeouts to a workqueue Christoph Hellwig
2015-11-23 20:31   ` Jeff Moyer
2015-11-23 20:48     ` Christoph Hellwig
2015-11-23 20:59       ` Jeff Moyer
2015-11-20 16:34 ` [PATCH 04/47] block: provide a new BLK_EH_QUIESCED timeout return value Christoph Hellwig
2015-11-24 15:16   ` Jeff Moyer
2015-11-24 15:40     ` Christoph Hellwig
2015-11-24 15:51       ` Jeff Moyer
2015-11-24 15:56         ` Christoph Hellwig
2015-11-24 16:34           ` Jeff Moyer
2015-11-24 16:47             ` Keith Busch
2015-11-24 17:56             ` Christoph Hellwig
2015-11-24 18:12               ` Jeff Moyer
2015-11-24 19:40                 ` Christoph Hellwig
2015-11-20 16:35 ` [PATCH 05/47] block: remoe REQ_ATOM_COMPLETE wrappers Christoph Hellwig
2015-11-23 21:23   ` Jeff Moyer
2015-11-24 21:22   ` Jens Axboe
2015-11-20 16:35 ` [PATCH 06/47] blk-mq: add a flags parameter to blk_mq_alloc_request Christoph Hellwig
2015-11-24 15:19   ` Jeff Moyer [this message]
2015-11-24 15:32     ` Christoph Hellwig
2015-11-24 21:21   ` Jens Axboe
2015-11-24 22:22     ` Christoph Hellwig
2015-11-24 22:25       ` Jens Axboe
2015-11-20 16:35 ` [PATCH 07/47] nvme: move struct nvme_iod to pci.c Christoph Hellwig
2015-11-20 16:35 ` [PATCH 08/47] nvme: split command submission helpers out of pci.c Christoph Hellwig
2015-11-20 16:35 ` [PATCH 09/47] nvme: add a vendor field to struct nvme_dev Christoph Hellwig
2015-11-20 16:35 ` [PATCH 10/47] nvme: use offset instead of a struct for registers Christoph Hellwig
2015-11-20 16:35 ` [PATCH 11/47] nvme: split a new struct nvme_ctrl out of struct nvme_dev Christoph Hellwig
2015-11-20 16:35 ` [PATCH 12/47] nvme: simplify nvme_setup_prps calling convention Christoph Hellwig
2015-11-20 16:35 ` [PATCH 13/47] nvme: refactor nvme_queue_rq Christoph Hellwig
2015-11-20 16:35 ` [PATCH 14/47] nvme: move nvme_error_status to common code Christoph Hellwig
2015-11-20 16:35 ` [PATCH 15/47] nvme: move nvme_setup_flush and nvme_setup_rw " Christoph Hellwig
2015-11-20 16:35 ` [PATCH 16/47] nvme: split __nvme_submit_sync_cmd Christoph Hellwig
2015-11-20 16:35 ` [PATCH 17/47] nvme: use the block layer for userspace passthrough metadata Christoph Hellwig
2015-11-20 16:35 ` [PATCH 18/47] nvme: move block_device_operations and ns/ctrl freeing to common code Christoph Hellwig
2015-11-20 16:35 ` [PATCH 19/47] nvme: add explicit quirk handling Christoph Hellwig
2015-11-20 16:35 ` [PATCH 20/47] nvme: add a common helper to read Identify Controller data Christoph Hellwig
2015-11-20 16:35 ` [PATCH 21/47] nvme: move the call to nvme_init_identify earlier Christoph Hellwig
2015-11-20 16:35 ` [PATCH 22/47] nvme: move namespace scanning to common code Christoph Hellwig
2015-11-20 16:35 ` [PATCH 23/47] nvme: move chardev and sysfs interface " Christoph Hellwig
2015-11-20 16:35 ` [PATCH 24/47] nvme: only add a controller to dev_list after it's been fully initialized Christoph Hellwig
2015-11-20 16:35 ` [PATCH 25/47] nvme: don't take the I/O queue q_lock in nvme_timeout Christoph Hellwig
2017-03-10 12:51   ` David Woodhouse
2017-03-10 14:24     ` Christoph Hellwig
2015-11-20 16:35 ` [PATCH 26/47] nvme: merge nvme_abort_req and nvme_timeout Christoph Hellwig
2015-11-20 16:35 ` [PATCH 27/47] nvme: do not restart the request timeout if we're resetting the controller Christoph Hellwig
2015-11-20 16:35 ` [PATCH 28/47] nvme: simplify resets Christoph Hellwig
2015-11-20 16:35 ` [PATCH 29/47] nvme: merge probe_work and reset_work Christoph Hellwig
2015-11-20 16:35 ` [PATCH 30/47] nvme: remove dead controllers from a work item Christoph Hellwig
2015-11-20 16:35 ` [PATCH 31/47] nvme: switch abort_limit to an atomic_t Christoph Hellwig
2015-11-20 16:35 ` [PATCH 32/47] NVMe: Implement namespace list scanning Christoph Hellwig
2015-11-20 16:35 ` [PATCH 33/47] NVMe: Use unbounded work queue for all work Christoph Hellwig
2015-11-20 16:35 ` [PATCH 34/47] NVMe: Remove device management handles on remove Christoph Hellwig
2015-11-20 16:50 ` NVMe mega patchbomb for Linux 4.5-rc Christoph Hellwig
2015-11-21  7:19 NVMe mega patchbomb for Linux 4.5-rc (resend) Christoph Hellwig
2015-11-21  7:19 ` [PATCH 06/47] blk-mq: add a flags parameter to blk_mq_alloc_request Christoph Hellwig

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=x49io4r2zmt.fsf@segfault.boston.devel.redhat.com \
    --to=jmoyer@redhat.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-NVME Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-nvme/0 linux-nvme/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-nvme linux-nvme/ https://lore.kernel.org/linux-nvme \
		linux-nvme@lists.infradead.org
	public-inbox-index linux-nvme

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.infradead.lists.linux-nvme


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git