[RFC PATCH 3/3] nvme: add the "debug" host driver

[Mikulas Patocka <mpatocka@redhat.com>]

This patch adds a new driver "nvme-debug". It uses memory as a backing
store and it is used to test the copy offload functionality.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
 drivers/nvme/host/Kconfig      |   13 
 drivers/nvme/host/Makefile     |    1 
 drivers/nvme/host/nvme-debug.c |  838 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 852 insertions(+)

Index: linux-2.6/drivers/nvme/host/Kconfig
===================================================================
--- linux-2.6.orig/drivers/nvme/host/Kconfig	2022-02-01 18:34:22.000000000 +0100
+++ linux-2.6/drivers/nvme/host/Kconfig	2022-02-01 18:34:22.000000000 +0100
@@ -83,3 +83,16 @@ config NVME_TCP
 	  from https://github.com/linux-nvme/nvme-cli.
 
 	  If unsure, say N.
+
+config NVME_DEBUG
+	tristate "NVM Express debug"
+	depends on INET
+	depends on BLOCK
+	select NVME_CORE
+	select NVME_FABRICS
+	select CRYPTO
+	select CRYPTO_CRC32C
+	help
+	  This pseudo driver simulates a NVMe adapter.
+
+	  If unsure, say N.
Index: linux-2.6/drivers/nvme/host/Makefile
===================================================================
--- linux-2.6.orig/drivers/nvme/host/Makefile	2022-02-01 18:34:22.000000000 +0100
+++ linux-2.6/drivers/nvme/host/Makefile	2022-02-01 18:34:22.000000000 +0100
@@ -8,6 +8,7 @@ obj-$(CONFIG_NVME_FABRICS)		+= nvme-fabr
 obj-$(CONFIG_NVME_RDMA)			+= nvme-rdma.o
 obj-$(CONFIG_NVME_FC)			+= nvme-fc.o
 obj-$(CONFIG_NVME_TCP)			+= nvme-tcp.o
+obj-$(CONFIG_NVME_DEBUG)		+= nvme-debug.o
 
 nvme-core-y				:= core.o ioctl.o
 nvme-core-$(CONFIG_TRACING)		+= trace.o
Index: linux-2.6/drivers/nvme/host/nvme-debug.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6/drivers/nvme/host/nvme-debug.c	2022-02-01 18:34:22.000000000 +0100
@@ -0,0 +1,838 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe debug
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/blk-mq.h>
+#include <linux/sort.h>
+#include <linux/version.h>
+
+#include "nvme.h"
+#include "fabrics.h"
+
+static ulong dev_size_mb = 16;
+module_param_named(dev_size_mb, dev_size_mb, ulong, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(dev_size_mb, "size in MiB of the namespace(def=8)");
+
+static unsigned sector_size = 512;
+module_param_named(sector_size, sector_size, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(sector_size, "logical block size in bytes (def=512)");
+
+struct nvme_debug_ctrl {
+	struct nvme_ctrl ctrl;
+	uint32_t reg_cc;
+	struct blk_mq_tag_set admin_tag_set;
+	struct blk_mq_tag_set io_tag_set;
+	struct workqueue_struct *admin_wq;
+	struct workqueue_struct *io_wq;
+	struct list_head namespaces;
+	struct list_head list;
+};
+
+struct nvme_debug_namespace {
+	struct list_head list;
+	uint32_t nsid;
+	unsigned char sector_size_bits;
+	size_t n_sectors;
+	void *space;
+	char uuid[16];
+};
+
+struct nvme_debug_request {
+	struct nvme_request req;
+	struct nvme_command cmd;
+	struct work_struct work;
+};
+
+static LIST_HEAD(nvme_debug_ctrl_list);
+static DEFINE_MUTEX(nvme_debug_ctrl_mutex);
+
+DEFINE_STATIC_PERCPU_RWSEM(nvme_debug_sem);
+
+static inline struct nvme_debug_ctrl *to_debug_ctrl(struct nvme_ctrl *nctrl)
+{
+	return container_of(nctrl, struct nvme_debug_ctrl, ctrl);
+}
+
+static struct nvme_debug_namespace *nvme_debug_find_namespace(struct nvme_debug_ctrl *ctrl, unsigned nsid)
+{
+	struct nvme_debug_namespace *ns;
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		if (ns->nsid == nsid)
+			return ns;
+	}
+	return NULL;
+}
+
+static bool nvme_debug_alloc_namespace(struct nvme_debug_ctrl *ctrl)
+{
+	struct nvme_debug_namespace *ns;
+	unsigned s;
+	size_t dsm;
+
+	ns = kmalloc(sizeof(struct nvme_debug_namespace), GFP_KERNEL);
+	if (!ns)
+		goto fail0;
+
+	ns->nsid = 1;
+	while (nvme_debug_find_namespace(ctrl, ns->nsid))
+		ns->nsid++;
+
+	s = READ_ONCE(sector_size);
+	if (s < 512 || s > PAGE_SIZE || !is_power_of_2(s))
+		goto fail1;
+	ns->sector_size_bits = __ffs(s);
+	dsm = READ_ONCE(dev_size_mb);
+	ns->n_sectors = dsm << (20 - ns->sector_size_bits);
+	if (ns->n_sectors >> (20 - ns->sector_size_bits) != dsm)
+		goto fail1;
+
+	if (ns->n_sectors << ns->sector_size_bits >> ns->sector_size_bits != ns->n_sectors)
+		goto fail1;
+
+	ns->space = vzalloc(ns->n_sectors << ns->sector_size_bits);
+	if (!ns->space)
+		goto fail1;
+
+	generate_random_uuid(ns->uuid);
+
+	list_add(&ns->list, &ctrl->namespaces);
+	return true;
+
+fail1:
+	kfree(ns);
+fail0:
+	return false;
+}
+
+static void nvme_debug_free_namespace(struct nvme_debug_namespace *ns)
+{
+	vfree(ns->space);
+	list_del(&ns->list);
+	kfree(ns);
+}
+
+static int nvme_debug_reg_read32(struct nvme_ctrl *nctrl, u32 off, u32 *val)
+{
+	struct nvme_debug_ctrl *ctrl = to_debug_ctrl(nctrl);
+	switch (off) {
+		case NVME_REG_VS: {
+			*val = 0x20000;
+			break;
+		}
+		case NVME_REG_CC: {
+			*val = ctrl->reg_cc;
+			break;
+		}
+		case NVME_REG_CSTS: {
+			*val = 0;
+			if (ctrl->reg_cc & NVME_CC_ENABLE)
+				*val |= NVME_CSTS_RDY;
+			if (ctrl->reg_cc & NVME_CC_SHN_MASK)
+				*val |= NVME_CSTS_SHST_CMPLT;
+			break;
+		}
+		default: {
+			printk("nvme_debug_reg_read32: %x\n", off);
+			return -ENOSYS;
+		}
+	}
+	return 0;
+}
+
+int nvme_debug_reg_read64(struct nvme_ctrl *nctrl, u32 off, u64 *val)
+{
+	switch (off) {
+		case NVME_REG_CAP: {
+			*val = (1ULL << 0) | (1ULL << 37);
+			break;
+		}
+		default: {
+			printk("nvme_debug_reg_read64: %x\n", off);
+			return -ENOSYS;
+		}
+	}
+	return 0;
+}
+
+int nvme_debug_reg_write32(struct nvme_ctrl *nctrl, u32 off, u32 val)
+{
+	struct nvme_debug_ctrl *ctrl = to_debug_ctrl(nctrl);
+	switch (off) {
+		case NVME_REG_CC: {
+			ctrl->reg_cc = val;
+			break;
+		}
+		default: {
+			printk("nvme_debug_reg_write32: %x, %x\n", off, val);
+			return -ENOSYS;
+		}
+	}
+	return 0;
+}
+
+static void nvme_debug_submit_async_event(struct nvme_ctrl *nctrl)
+{
+	printk("nvme_debug_submit_async_event\n");
+}
+
+static void nvme_debug_delete_ctrl(struct nvme_ctrl *nctrl)
+{
+	nvme_shutdown_ctrl(nctrl);
+}
+
+static void nvme_debug_free_namespaces(struct nvme_debug_ctrl *ctrl)
+{
+	if (!list_empty(&ctrl->namespaces)) {
+		struct nvme_debug_namespace *ns = container_of(ctrl->namespaces.next, struct nvme_debug_namespace, list);
+		nvme_debug_free_namespace(ns);
+	}
+}
+
+static void nvme_debug_free_ctrl(struct nvme_ctrl *nctrl)
+{
+	struct nvme_debug_ctrl *ctrl = to_debug_ctrl(nctrl);
+
+	flush_workqueue(ctrl->admin_wq);
+	flush_workqueue(ctrl->io_wq);
+
+	nvme_debug_free_namespaces(ctrl);
+
+	if (list_empty(&ctrl->list))
+		goto free_ctrl;
+
+	mutex_lock(&nvme_debug_ctrl_mutex);
+	list_del(&ctrl->list);
+	mutex_unlock(&nvme_debug_ctrl_mutex);
+
+	nvmf_free_options(nctrl->opts);
+free_ctrl:
+	destroy_workqueue(ctrl->admin_wq);
+	destroy_workqueue(ctrl->io_wq);
+	kfree(ctrl);
+}
+
+static int nvme_debug_get_address(struct nvme_ctrl *nctrl, char *buf, int size)
+{
+	int len = 0;
+	len += snprintf(buf, size, "debug");
+	return len;
+}
+
+static void nvme_debug_reset_ctrl_work(struct work_struct *work)
+{
+	printk("nvme_reset_ctrl_work\n");
+}
+
+static void copy_data_request(struct request *req, void *data, size_t size, bool to_req)
+{
+	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
+		void *addr;
+		struct bio_vec bv = req->special_vec;
+		addr = kmap_atomic(bv.bv_page);
+		if (to_req) {
+			memcpy(addr + bv.bv_offset, data, bv.bv_len);
+			flush_dcache_page(bv.bv_page);
+		} else {
+			flush_dcache_page(bv.bv_page);
+			memcpy(data, addr + bv.bv_offset, bv.bv_len);
+		}
+		kunmap_atomic(addr);
+		data += bv.bv_len;
+		size -= bv.bv_len;
+	} else {
+		struct req_iterator bi;
+		struct bio_vec bv;
+		rq_for_each_segment(bv, req, bi) {
+			void *addr;
+			addr = kmap_atomic(bv.bv_page);
+			if (to_req) {
+				memcpy(addr + bv.bv_offset, data, bv.bv_len);
+				flush_dcache_page(bv.bv_page);
+			} else {
+				flush_dcache_page(bv.bv_page);
+				memcpy(data, addr + bv.bv_offset, bv.bv_len);
+			}
+			kunmap_atomic(addr);
+			data += bv.bv_len;
+			size -= bv.bv_len;
+		}
+	}
+	if (size)
+		printk("size mismatch: %lx\n", (unsigned long)size);
+}
+
+static void nvme_debug_identify_ns(struct nvme_debug_ctrl *ctrl, struct request *req)
+{
+	struct nvme_id_ns *id;
+	struct nvme_debug_namespace *ns;
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+
+	id = kzalloc(sizeof(struct nvme_id_ns), GFP_NOIO);
+	if (!id) {
+		blk_mq_end_request(req, BLK_STS_RESOURCE);
+		return;
+	}
+
+	ns = nvme_debug_find_namespace(ctrl, le32_to_cpu(ndr->cmd.identify.nsid));
+	if (!ns) {
+		nvme_req(req)->status = cpu_to_le16(NVME_SC_INVALID_NS);
+		goto free_ret;
+	}
+
+	id->nsze = cpu_to_le64(ns->n_sectors);
+	id->ncap = id->nsze;
+	id->nuse = id->nsze;
+	/*id->nlbaf = 0;*/
+	id->dlfeat = 0x01;
+	id->lbaf[0].ds = ns->sector_size_bits;
+
+	copy_data_request(req, id, sizeof(struct nvme_id_ns), true);
+
+free_ret:
+	kfree(id);
+	blk_mq_end_request(req, BLK_STS_OK);
+}
+
+static void nvme_debug_identify_ctrl(struct nvme_debug_ctrl *ctrl, struct request *req)
+{
+	struct nvme_debug_namespace *ns;
+	struct nvme_id_ctrl *id;
+	char ver[9];
+	size_t ver_len;
+
+	id = kzalloc(sizeof(struct nvme_id_ctrl), GFP_NOIO);
+	if (!id) {
+		blk_mq_end_request(req, BLK_STS_RESOURCE);
+		return;
+	}
+
+	id->vid = cpu_to_le16(PCI_VENDOR_ID_REDHAT);
+	id->ssvid = cpu_to_le16(PCI_VENDOR_ID_REDHAT);
+	memset(id->sn, ' ', sizeof id->sn);
+	memset(id->mn, ' ', sizeof id->mn);
+	memcpy(id->mn, "nvme-debug", 10);
+	snprintf(ver, sizeof ver, "%X", LINUX_VERSION_CODE);
+	ver_len = min(strlen(ver), sizeof id->fr);
+	memset(id->fr, ' ', sizeof id->fr);
+	memcpy(id->fr, ver, ver_len);
+	memcpy(id->ieee, "\xe9\xf2\x40", sizeof id->ieee);
+	id->ver = cpu_to_le32(0x20000);
+	id->kas = cpu_to_le16(100);
+	id->sqes = 0x66;
+	id->cqes = 0x44;
+	id->maxcmd = cpu_to_le16(1);
+	mutex_lock(&nvme_debug_ctrl_mutex);
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		if (ns->nsid > le32_to_cpu(id->nn))
+			id->nn = cpu_to_le32(ns->nsid);
+	}
+	mutex_unlock(&nvme_debug_ctrl_mutex);
+	id->oncs = cpu_to_le16(NVME_CTRL_ONCS_COPY);
+	id->vwc = 0x6;
+	id->mnan = cpu_to_le32(0xffffffff);
+	strcpy(id->subnqn, "nqn.2021-09.com.redhat:nvme-debug");
+	id->ioccsz = cpu_to_le32(4);
+	id->iorcsz = cpu_to_le32(1);
+
+	copy_data_request(req, id, sizeof(struct nvme_id_ctrl), true);
+
+	kfree(id);
+	blk_mq_end_request(req, BLK_STS_OK);
+}
+
+static int cmp_ns(const void *a1, const void *a2)
+{
+	__u32 v1 = le32_to_cpu(*(__u32 *)a1);
+	__u32 v2 = le32_to_cpu(*(__u32 *)a2);
+	if (!v1)
+		v1 = 0xffffffffU;
+	if (!v2)
+		v2 = 0xffffffffU;
+	if (v1 < v2)
+		return -1;
+	if (v1 > v2)
+		return 1;
+	return 0;
+}
+
+static void nvme_debug_identify_active_ns(struct nvme_debug_ctrl *ctrl, struct request *req)
+{
+	struct nvme_debug_namespace *ns;
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+	unsigned size;
+	__u32 *id;
+	unsigned idp;
+
+	if (le32_to_cpu(ndr->cmd.identify.nsid) >= 0xFFFFFFFE) {
+		nvme_req(req)->status = cpu_to_le16(NVME_SC_INVALID_NS);
+		blk_mq_end_request(req, BLK_STS_OK);
+		return;
+	}
+
+	mutex_lock(&nvme_debug_ctrl_mutex);
+	size = 0;
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		size++;
+	}
+	size = min(size, 1024U);
+
+	id = kzalloc(sizeof(__u32) * size, GFP_NOIO);
+	if (!id) {
+		mutex_unlock(&nvme_debug_ctrl_mutex);
+		blk_mq_end_request(req, BLK_STS_RESOURCE);
+		return;
+	}
+
+	idp = 0;
+	list_for_each_entry(ns, &ctrl->namespaces, list) {
+		if (ns->nsid > le32_to_cpu(ndr->cmd.identify.nsid))
+			id[idp++] = cpu_to_le32(ns->nsid);
+	}
+	mutex_unlock(&nvme_debug_ctrl_mutex);
+	sort(id, idp, sizeof(__u32), cmp_ns, NULL);
+
+	copy_data_request(req, id, sizeof(__u32) * 1024, true);
+
+	kfree(id);
+	blk_mq_end_request(req, BLK_STS_OK);
+}
+
+static void nvme_debug_identify_ns_desc_list(struct nvme_debug_ctrl *ctrl, struct request *req)
+{
+	struct nvme_ns_id_desc *id;
+	struct nvme_debug_namespace *ns;
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+	id = kzalloc(4096, GFP_NOIO);
+	if (!id) {
+		blk_mq_end_request(req, BLK_STS_RESOURCE);
+		return;
+	}
+
+	ns = nvme_debug_find_namespace(ctrl, le32_to_cpu(ndr->cmd.identify.nsid));
+	if (!ns) {
+		nvme_req(req)->status = cpu_to_le16(NVME_SC_INVALID_NS);
+		goto free_ret;
+	}
+
+	id->nidt = NVME_NIDT_UUID;
+	id->nidl = NVME_NIDT_UUID_LEN;
+	memcpy((char *)(id + 1), ns->uuid, NVME_NIDT_UUID_LEN);
+
+	copy_data_request(req, id, 4096, true);
+
+free_ret:
+	kfree(id);
+	blk_mq_end_request(req, BLK_STS_OK);
+}
+
+static void nvme_debug_identify_ctrl_cs(struct request *req)
+{
+	struct nvme_id_ctrl_nvm *id;
+	id = kzalloc(sizeof(struct nvme_id_ctrl_nvm), GFP_NOIO);
+	if (!id) {
+		blk_mq_end_request(req, BLK_STS_RESOURCE);
+		return;
+	}
+
+	copy_data_request(req, id, sizeof(struct nvme_id_ctrl_nvm), true);
+
+	kfree(id);
+	blk_mq_end_request(req, BLK_STS_OK);
+}
+
+static void nvme_debug_admin_rq(struct work_struct *w)
+{
+	struct nvme_debug_request *ndr = container_of(w, struct nvme_debug_request, work);
+	struct request *req = (struct request *)ndr - 1;
+	struct nvme_debug_ctrl *ctrl = to_debug_ctrl(ndr->req.ctrl);
+
+	switch (ndr->cmd.common.opcode) {
+		case nvme_admin_identify: {
+			switch (ndr->cmd.identify.cns) {
+				case NVME_ID_CNS_NS: {
+					percpu_down_read(&nvme_debug_sem);
+					nvme_debug_identify_ns(ctrl, req);
+					percpu_up_read(&nvme_debug_sem);
+					return;
+				};
+				case NVME_ID_CNS_CTRL: {
+					percpu_down_read(&nvme_debug_sem);
+					nvme_debug_identify_ctrl(ctrl, req);
+					percpu_up_read(&nvme_debug_sem);
+					return;
+				}
+				case NVME_ID_CNS_NS_ACTIVE_LIST: {
+					percpu_down_read(&nvme_debug_sem);
+					nvme_debug_identify_active_ns(ctrl, req);
+					percpu_up_read(&nvme_debug_sem);
+					return;
+				}
+				case NVME_ID_CNS_NS_DESC_LIST: {
+					percpu_down_read(&nvme_debug_sem);
+					nvme_debug_identify_ns_desc_list(ctrl, req);
+					percpu_up_read(&nvme_debug_sem);
+					return;
+				}
+				case NVME_ID_CNS_CS_CTRL: {
+					percpu_down_read(&nvme_debug_sem);
+					nvme_debug_identify_ctrl_cs(req);
+					percpu_up_read(&nvme_debug_sem);
+					return;
+				}
+				default: {
+					printk("nvme_admin_identify: %x\n", ndr->cmd.identify.cns);
+					break;
+				}
+			}
+			break;
+		}
+		default: {
+			printk("nvme_debug_admin_rq: %x\n", ndr->cmd.common.opcode);
+			break;
+		}
+	}
+	blk_mq_end_request(req, BLK_STS_NOTSUPP);
+}
+
+static void nvme_debug_rw(struct nvme_debug_namespace *ns, struct request *req)
+{
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+	__u64 lba = cpu_to_le64(ndr->cmd.rw.slba);
+	__u64 len = (__u64)cpu_to_le16(ndr->cmd.rw.length) + 1;
+	void *addr;
+	if (unlikely(lba + len < lba) || unlikely(lba + len > ns->n_sectors)) {
+		blk_mq_end_request(req, BLK_STS_NOTSUPP);
+		return;
+	}
+	addr = ns->space + (lba << ns->sector_size_bits);
+	copy_data_request(req, addr, len << ns->sector_size_bits, ndr->cmd.rw.opcode == nvme_cmd_read);
+	blk_mq_end_request(req, BLK_STS_OK);
+}
+
+static void nvme_debug_copy(struct nvme_debug_namespace *ns, struct request *req)
+{
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+	__u64 dlba = cpu_to_le64(ndr->cmd.copy.sdlba);
+	unsigned n_descs = ndr->cmd.copy.length + 1;
+	struct nvme_copy_desc *descs;
+	unsigned i, ret;
+
+	descs = kmalloc(sizeof(struct nvme_copy_desc) * n_descs, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
+	if (!descs) {
+		blk_mq_end_request(req, BLK_STS_RESOURCE);
+		return;
+	}
+
+	copy_data_request(req, descs, sizeof(struct nvme_copy_desc) * n_descs, false);
+
+	for (i = 0; i < n_descs; i++) {
+		struct nvme_copy_desc *desc = &descs[i];
+		__u64 slba = cpu_to_le64(desc->slba);
+		__u64 len = (__u64)cpu_to_le16(desc->length) + 1;
+		void *saddr, *daddr;
+
+		if (unlikely(slba + len < slba) || unlikely(slba + len > ns->n_sectors) ||
+		    unlikely(dlba + len < dlba) || unlikely(dlba + len > ns->n_sectors)) {
+			ret = BLK_STS_NOTSUPP;
+			goto free_ret;
+		}
+
+		saddr = ns->space + (slba << ns->sector_size_bits);
+		daddr = ns->space + (dlba << ns->sector_size_bits);
+
+		memcpy(daddr, saddr, len << ns->sector_size_bits);
+
+		dlba += len;
+	}
+
+	ret = BLK_STS_OK;
+
+free_ret:
+	kfree(descs);
+
+	blk_mq_end_request(req, ret);
+}
+
+static void nvme_debug_io_rq(struct work_struct *w)
+{
+	struct nvme_debug_request *ndr = container_of(w, struct nvme_debug_request, work);
+	struct request *req = (struct request *)ndr - 1;
+	struct nvme_debug_ctrl *ctrl = to_debug_ctrl(ndr->req.ctrl);
+	__u32 nsid = le32_to_cpu(ndr->cmd.common.nsid);
+	struct nvme_debug_namespace *ns;
+
+	percpu_down_read(&nvme_debug_sem);
+	ns = nvme_debug_find_namespace(ctrl, nsid);
+	if (unlikely(!ns))
+		goto ret_notsupp;
+
+	switch (ndr->cmd.common.opcode) {
+		case nvme_cmd_flush: {
+			blk_mq_end_request(req, BLK_STS_OK);
+			goto ret;
+		}
+		case nvme_cmd_read:
+		case nvme_cmd_write: {
+			nvme_debug_rw(ns, req);
+			goto ret;
+		}
+		case nvme_cmd_copy: {
+			nvme_debug_copy(ns, req);
+			goto ret;
+		}
+		default: {
+			printk("nvme_debug_io_rq: %x\n", ndr->cmd.common.opcode);
+			break;
+		}
+	}
+ret_notsupp:
+	blk_mq_end_request(req, BLK_STS_NOTSUPP);
+ret:
+	percpu_up_read(&nvme_debug_sem);
+}
+
+static blk_status_t nvme_debug_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
+{
+	struct request *req = bd->rq;
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+	struct nvme_debug_ctrl *ctrl = to_debug_ctrl(ndr->req.ctrl);
+	struct nvme_ns *ns = hctx->queue->queuedata;
+	blk_status_t r;
+
+	r = nvme_setup_cmd(ns, req);
+	if (unlikely(r))
+		return r;
+
+	if (!ns) {
+		INIT_WORK(&ndr->work, nvme_debug_admin_rq);
+		queue_work(ctrl->admin_wq, &ndr->work);
+		return BLK_STS_OK;
+	} else if (unlikely((req->cmd_flags & REQ_OP_MASK) == REQ_OP_COPY_READ_TOKEN)) {
+		blk_mq_end_request(req, BLK_STS_OK);
+		return BLK_STS_OK;
+	} else {
+		INIT_WORK(&ndr->work, nvme_debug_io_rq);
+		queue_work(ctrl->io_wq, &ndr->work);
+		return BLK_STS_OK;
+	}
+}
+
+static int nvme_debug_init_request(struct blk_mq_tag_set *set, struct request *req, unsigned hctx_idx, unsigned numa_node)
+{
+	struct nvme_debug_ctrl *ctrl = set->driver_data;
+	struct nvme_debug_request *ndr = blk_mq_rq_to_pdu(req);
+	nvme_req(req)->ctrl = &ctrl->ctrl;
+	nvme_req(req)->cmd = &ndr->cmd;
+	return 0;
+}
+
+static int nvme_debug_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned hctx_idx)
+{
+	struct nvme_debug_ctrl *ctrl = data;
+	hctx->driver_data = ctrl;
+	return 0;
+}
+
+static const struct blk_mq_ops nvme_debug_mq_ops = {
+	.queue_rq = nvme_debug_queue_rq,
+	.init_request = nvme_debug_init_request,
+	.init_hctx = nvme_debug_init_hctx,
+};
+
+static int nvme_debug_configure_admin_queue(struct nvme_debug_ctrl *ctrl)
+{
+	int r;
+
+	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
+	ctrl->admin_tag_set.ops = &nvme_debug_mq_ops;
+	ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
+	ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS;
+	ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node;
+	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_debug_request);
+	ctrl->admin_tag_set.driver_data = ctrl;
+	ctrl->admin_tag_set.nr_hw_queues = 1;
+	ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT;
+	ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;
+
+	r = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
+	if (r)
+		goto ret0;
+	ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
+
+	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
+	if (IS_ERR(ctrl->ctrl.admin_q)) {
+		r = PTR_ERR(ctrl->ctrl.admin_q);
+		goto ret1;
+	}
+
+	r = nvme_enable_ctrl(&ctrl->ctrl);
+	if (r)
+		goto ret2;
+
+	nvme_start_admin_queue(&ctrl->ctrl);
+
+	r = nvme_init_ctrl_finish(&ctrl->ctrl);
+	if (r)
+		goto ret3;
+
+	return 0;
+
+ret3:
+	nvme_stop_admin_queue(&ctrl->ctrl);
+	blk_sync_queue(ctrl->ctrl.admin_q);
+ret2:
+	blk_cleanup_queue(ctrl->ctrl.admin_q);
+ret1:
+	blk_mq_free_tag_set(&ctrl->admin_tag_set);
+ret0:
+	return r;
+}
+
+static int nvme_debug_configure_io_queue(struct nvme_debug_ctrl *ctrl)
+{
+	int r;
+
+	memset(&ctrl->io_tag_set, 0, sizeof(ctrl->io_tag_set));
+	ctrl->io_tag_set.ops = &nvme_debug_mq_ops;
+	ctrl->io_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
+	ctrl->io_tag_set.reserved_tags = NVMF_RESERVED_TAGS;
+	ctrl->io_tag_set.numa_node = ctrl->ctrl.numa_node;
+	ctrl->io_tag_set.cmd_size = sizeof(struct nvme_debug_request);
+	ctrl->io_tag_set.driver_data = ctrl;
+	ctrl->io_tag_set.nr_hw_queues = 1;
+	ctrl->io_tag_set.timeout = NVME_ADMIN_TIMEOUT;
+	ctrl->io_tag_set.flags = BLK_MQ_F_NO_SCHED;
+
+	r = blk_mq_alloc_tag_set(&ctrl->io_tag_set);
+	if (r)
+		goto ret0;
+	ctrl->ctrl.tagset = &ctrl->io_tag_set;
+	return 0;
+
+ret0:
+	return r;
+}
+
+static const struct nvme_ctrl_ops nvme_debug_ctrl_ops = {
+	.name = "debug",
+	.module = THIS_MODULE,
+	.flags = NVME_F_FABRICS,
+	.reg_read32 = nvme_debug_reg_read32,
+	.reg_read64 = nvme_debug_reg_read64,
+	.reg_write32 = nvme_debug_reg_write32,
+	.free_ctrl = nvme_debug_free_ctrl,
+	.submit_async_event = nvme_debug_submit_async_event,
+	.delete_ctrl = nvme_debug_delete_ctrl,
+	.get_address = nvme_debug_get_address,
+};
+
+static struct nvme_ctrl *nvme_debug_create_ctrl(struct device *dev,
+		struct nvmf_ctrl_options *opts)
+{
+	int r;
+	struct nvme_debug_ctrl *ctrl;
+
+	ctrl = kzalloc(sizeof(struct nvme_debug_ctrl), GFP_KERNEL);
+	if (!ctrl) {
+		r = -ENOMEM;
+		goto ret0;
+	}
+
+	INIT_LIST_HEAD(&ctrl->list);
+	INIT_LIST_HEAD(&ctrl->namespaces);
+	ctrl->ctrl.opts = opts;
+	ctrl->ctrl.queue_count = 2;
+	INIT_WORK(&ctrl->ctrl.reset_work, nvme_debug_reset_ctrl_work);
+
+	ctrl->admin_wq = alloc_workqueue("nvme-debug-admin", WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
+	if (!ctrl->admin_wq)
+		goto ret1;
+
+	ctrl->io_wq = alloc_workqueue("nvme-debug-io", WQ_MEM_RECLAIM, 0);
+	if (!ctrl->io_wq)
+		goto ret1;
+
+	if (!nvme_debug_alloc_namespace(ctrl)) {
+		r = -ENOMEM;
+		goto ret1;
+	}
+
+	r = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_debug_ctrl_ops, 0);
+	if (r)
+		goto ret1;
+
+        if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
+		goto ret2;
+
+	r = nvme_debug_configure_admin_queue(ctrl);
+	if (r)
+		goto ret2;
+
+	r = nvme_debug_configure_io_queue(ctrl);
+	if (r)
+		goto ret2;
+
+        if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
+		goto ret2;
+
+	nvme_start_ctrl(&ctrl->ctrl);
+
+	mutex_lock(&nvme_debug_ctrl_mutex);
+	list_add_tail(&ctrl->list, &nvme_debug_ctrl_list);
+	mutex_unlock(&nvme_debug_ctrl_mutex);
+
+	return &ctrl->ctrl;
+
+ret2:
+	nvme_uninit_ctrl(&ctrl->ctrl);
+	nvme_put_ctrl(&ctrl->ctrl);
+	return ERR_PTR(r);
+ret1:
+	nvme_debug_free_namespaces(ctrl);
+	if (ctrl->admin_wq)
+		destroy_workqueue(ctrl->admin_wq);
+	if (ctrl->io_wq)
+		destroy_workqueue(ctrl->io_wq);
+	kfree(ctrl);
+ret0:
+	return ERR_PTR(r);
+}
+
+static struct nvmf_transport_ops nvme_debug_transport = {
+	.name		= "debug",
+	.module		= THIS_MODULE,
+	.required_opts	= NVMF_OPT_TRADDR,
+	.allowed_opts	= NVMF_OPT_CTRL_LOSS_TMO,
+	.create_ctrl	= nvme_debug_create_ctrl,
+};
+
+static int __init nvme_debug_init_module(void)
+{
+	nvmf_register_transport(&nvme_debug_transport);
+	return 0;
+}
+
+static void __exit nvme_debug_cleanup_module(void)
+{
+	struct nvme_debug_ctrl *ctrl;
+
+	nvmf_unregister_transport(&nvme_debug_transport);
+
+	mutex_lock(&nvme_debug_ctrl_mutex);
+	list_for_each_entry(ctrl, &nvme_debug_ctrl_list, list) {
+		nvme_delete_ctrl(&ctrl->ctrl);
+	}
+	mutex_unlock(&nvme_debug_ctrl_mutex);
+	flush_workqueue(nvme_delete_wq);
+}
+
+module_init(nvme_debug_init_module);
+module_exit(nvme_debug_cleanup_module);
+
+MODULE_LICENSE("GPL v2");