[PATCH V3] nvme: enable char device per namespace

[PATCH V3] nvme: enable char device per namespace

[javier]

From: Javier González <javier.gonz@samsung.com>

Create a char device per NVMe namespace. This char device is always
initialized, independently of whether the features implemented by the
device are supported by the kernel. User-space can therefore always
issue IOCTLs to the NVMe driver using the char device.

The char device is presented as /dev/generic-nvmeXcYnZ. This naming
scheme follows the convention of the hidden device (nvmeXcYnZ). Support
for multipath will follow.

Christoph, Keith: Is this going in the right direction?

Keith: Regarding nvme-cli support, what do you think about reporting the
char device as existing block devices? If this is OK with you, I will
submit patches for the filters.

Changes since V2:
  - Apply a number of naming and code structure improvements (from
    Christoph)
  - Use i_cdev to pull struct nvme_ns in the ioctl path instead of
    populating file->private_data (from Christoph)
  - Change char device and sysfs entries to /dev/generic-nvmeXcYnZ to
    follow the hidden device naming scheme (from Christoph and Keith)

Changes since V1:
  - Remove patches 1-3 which are already picked up by Christoph
  - Change the char device and sysfs entries to nvmeXnYc / c signals
    char device
  - Address Minwoo's comments on inline functions and style

Signed-off-by: Javier González <javier.gonz@samsung.com>
---
 drivers/nvme/host/core.c | 142 ++++++++++++++++++++++++++++++++++-----
 drivers/nvme/host/nvme.h |   8 +++
 2 files changed, 134 insertions(+), 16 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 99f91efe3824..006ac8ee9c7c 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -86,7 +86,9 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
+static dev_t nvme_ns_base_chr_devt;
 static struct class *nvme_class;
+static struct class *nvme_ns_class;
 static struct class *nvme_subsys_class;
 
 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
@@ -538,6 +540,7 @@ static void nvme_free_ns(struct kref *kref)
 	if (ns->ndev)
 		nvme_nvm_unregister(ns);
 
+	cdev_device_del(&ns->cdev, &ns->cdev_device);
 	put_disk(ns->disk);
 	nvme_put_ns_head(ns->head);
 	nvme_put_ctrl(ns->ctrl);
@@ -1738,15 +1741,15 @@ static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
 	return ret;
 }
 
-static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
-		unsigned int cmd, unsigned long arg)
+static int nvme_disk_ioctl(struct gendisk *disk, unsigned int cmd,
+			   unsigned long arg)
 {
 	struct nvme_ns_head *head = NULL;
 	void __user *argp = (void __user *)arg;
 	struct nvme_ns *ns;
 	int srcu_idx, ret;
 
-	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
+	ns = nvme_get_ns_from_disk(disk, &head, &srcu_idx);
 	if (unlikely(!ns))
 		return -EWOULDBLOCK;
 
@@ -1783,6 +1786,12 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
 	return ret;
 }
 
+static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
+		      unsigned int cmd, unsigned long arg)
+{
+	return nvme_disk_ioctl(bdev->bd_disk, cmd, arg);
+}
+
 #ifdef CONFIG_COMPAT
 struct nvme_user_io32 {
 	__u8	opcode;
@@ -1824,10 +1833,8 @@ static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
 #define nvme_compat_ioctl	NULL
 #endif /* CONFIG_COMPAT */
 
-static int nvme_open(struct block_device *bdev, fmode_t mode)
+static int nvme_ns_open(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = bdev->bd_disk->private_data;
-
 #ifdef CONFIG_NVME_MULTIPATH
 	/* should never be called due to GENHD_FL_HIDDEN */
 	if (WARN_ON_ONCE(ns->head->disk))
@@ -1846,14 +1853,22 @@ static int nvme_open(struct block_device *bdev, fmode_t mode)
 	return -ENXIO;
 }
 
-static void nvme_release(struct gendisk *disk, fmode_t mode)
+static void nvme_ns_release(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = disk->private_data;
-
 	module_put(ns->ctrl->ops->module);
 	nvme_put_ns(ns);
 }
 
+static int nvme_open(struct block_device *bdev, fmode_t mode)
+{
+	return nvme_ns_open(bdev->bd_disk->private_data);
+}
+
+static void nvme_release(struct gendisk *disk, fmode_t mode)
+{
+	nvme_ns_release(disk->private_data);
+}
+
 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 {
 	/* some standard values */
@@ -2209,6 +2224,13 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
 	return 0;
 
 out_unfreeze:
+	/*
+	 * When the device does not support any of the features required by the
+	 * kernel (or viceversa), hide the block device. We can still rely on
+	 * the namespace char device for submitting IOCTLs
+	 */
+	ns->disk->flags |= GENHD_FL_HIDDEN;
+
 	blk_mq_unfreeze_queue(ns->disk->queue);
 	return ret;
 }
@@ -2346,6 +2368,38 @@ static const struct block_device_operations nvme_bdev_ops = {
 	.pr_ops		= &nvme_pr_ops,
 };
 
+static int nvme_cdev_open(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	return nvme_ns_open(ns);
+}
+
+static int nvme_cdev_release(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	nvme_ns_release(ns);
+	return 0;
+}
+
+static long nvme_cdev_ioctl(struct file *file, unsigned int cmd,
+			    unsigned long arg)
+{
+	struct nvme_ns *ns = container_of(file->f_inode->i_cdev,
+				struct nvme_ns, cdev);
+
+	return nvme_disk_ioctl(ns->disk, cmd, arg);
+}
+
+static const struct file_operations nvme_cdev_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvme_cdev_open,
+	.release	= nvme_cdev_release,
+	.unlocked_ioctl	= nvme_cdev_ioctl,
+	.compat_ioctl	= compat_ptr_ioctl,
+};
+
 #ifdef CONFIG_NVME_MULTIPATH
 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
 {
@@ -3343,6 +3397,9 @@ static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
 {
 	struct gendisk *disk = dev_to_disk(dev);
 
+	if (dev->class == nvme_ns_class)
+		return nvme_get_ns_from_cdev(dev)->head;
+
 	if (disk->fops == &nvme_bdev_ops)
 		return nvme_get_ns_from_dev(dev)->head;
 	else
@@ -3474,6 +3531,11 @@ const struct attribute_group *nvme_ns_id_attr_groups[] = {
 	NULL,
 };
 
+const struct attribute_group *nvme_ns_char_id_attr_groups[] = {
+	&nvme_ns_id_attr_group,
+	NULL,
+};
+
 #define nvme_show_str_function(field)						\
 static ssize_t  field##_show(struct device *dev,				\
 			    struct device_attribute *attr, char *buf)		\
@@ -3866,6 +3928,36 @@ struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 }
 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
 
+static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
+{
+	char cdisk_name[DISK_NAME_LEN];
+	int ret;
+
+	device_initialize(&ns->cdev_device);
+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
+				     ns->head->instance);
+	ns->cdev_device.class = nvme_ns_class;
+	ns->cdev_device.parent = ctrl->device;
+	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
+	dev_set_drvdata(&ns->cdev_device, ns);
+
+	sprintf(cdisk_name, "nvme-generic-%dc%dn%d", ctrl->subsys->instance,
+		ctrl->instance, ns->head->instance);
+
+	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
+	if (ret)
+		return ret;
+
+	cdev_init(&ns->cdev, &nvme_cdev_fops);
+	ns->cdev.owner = ctrl->ops->module;
+
+	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
+	if (ret)
+		kfree_const(ns->cdev_device.kobj.name);
+
+	return ret;
+}
+
 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 		struct nvme_ns_ids *ids)
 {
@@ -3912,8 +4004,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
 	ns->disk = disk;
 
-	if (nvme_update_ns_info(ns, id))
-		goto out_put_disk;
+	nvme_update_ns_info(ns, id);
 
 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
 		if (nvme_nvm_register(ns, disk_name, node)) {
@@ -3929,9 +4020,12 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	nvme_get_ctrl(ctrl);
 
 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
-
 	nvme_mpath_add_disk(ns, id);
 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
+
+	if (nvme_alloc_chardev_ns(ctrl, ns))
+		goto out_put_disk;
+
 	kfree(id);
 
 	return;
@@ -4733,23 +4827,38 @@ static int __init nvme_core_init(void)
 	if (result < 0)
 		goto destroy_delete_wq;
 
+	result = alloc_chrdev_region(&nvme_ns_base_chr_devt, 0,
+			NVME_MINORS, "nvmec");
+	if (result < 0)
+		goto unregister_dev_chrdev;
+
 	nvme_class = class_create(THIS_MODULE, "nvme");
 	if (IS_ERR(nvme_class)) {
 		result = PTR_ERR(nvme_class);
-		goto unregister_chrdev;
+		goto unregister_ns_chrdev;
 	}
 	nvme_class->dev_uevent = nvme_class_uevent;
 
+	nvme_ns_class = class_create(THIS_MODULE, "nvme-ns");
+	if (IS_ERR(nvme_ns_class)) {
+		result = PTR_ERR(nvme_ns_class);
+		goto destroy_dev_class;
+	}
+
 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
 	if (IS_ERR(nvme_subsys_class)) {
 		result = PTR_ERR(nvme_subsys_class);
-		goto destroy_class;
+		goto destroy_ns_class;
 	}
 	return 0;
 
-destroy_class:
+destroy_ns_class:
+	class_destroy(nvme_ns_class);
+destroy_dev_class:
 	class_destroy(nvme_class);
-unregister_chrdev:
+unregister_ns_chrdev:
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
+unregister_dev_chrdev:
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 destroy_delete_wq:
 	destroy_workqueue(nvme_delete_wq);
@@ -4765,6 +4874,7 @@ static void __exit nvme_core_exit(void)
 {
 	class_destroy(nvme_subsys_class);
 	class_destroy(nvme_class);
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 	destroy_workqueue(nvme_delete_wq);
 	destroy_workqueue(nvme_reset_wq);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index bfcedfa4b057..1dd99f207aee 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -439,6 +439,9 @@ struct nvme_ns {
 	struct kref kref;
 	struct nvme_ns_head *head;
 
+	struct device cdev_device;	/* char device */
+	struct cdev cdev;
+
 	int lba_shift;
 	u16 ms;
 	u16 sgs;
@@ -818,6 +821,11 @@ static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
 	return dev_to_disk(dev)->private_data;
 }
 
+static inline struct nvme_ns *nvme_get_ns_from_cdev(struct device *dev)
+{
+	return dev_get_drvdata(dev);
+}
+
 #ifdef CONFIG_NVME_HWMON
 int nvme_hwmon_init(struct nvme_ctrl *ctrl);
 #else
-- 
2.17.1

[PATCH V3] nvme: enable char device per namespace

[javier]

From: Javier González <javier.gonz@samsung.com>

Create a char device per NVMe namespace. This char device is always
initialized, independently of whether the features implemented by the
device are supported by the kernel. User-space can therefore always
issue IOCTLs to the NVMe driver using the char device.

The char device is presented as /dev/generic-nvmeXcYnZ. This naming
scheme follows the convention of the hidden device (nvmeXcYnZ). Support
for multipath will follow.

Christoph, Keith: Is this going in the right direction?

Keith: Regarding nvme-cli support, what do you think about reporting the
char device as existing block devices? If this is OK with you, I will
submit patches for the filters.

Changes since V2:
  - Apply a number of naming and code structure improvements (from
    Christoph)
  - Use i_cdev to pull struct nvme_ns in the ioctl path instead of
    populating file->private_data (from Christoph)
  - Change char device and sysfs entries to /dev/generic-nvmeXcYnZ to
    follow the hidden device naming scheme (from Christoph and Keith)

Changes since V1:
  - Remove patches 1-3 which are already picked up by Christoph
  - Change the char device and sysfs entries to nvmeXnYc / c signals
    char device
  - Address Minwoo's comments on inline functions and style

Signed-off-by: Javier González <javier.gonz@samsung.com>
---
 drivers/nvme/host/core.c | 142 ++++++++++++++++++++++++++++++++++-----
 drivers/nvme/host/nvme.h |   8 +++
 2 files changed, 134 insertions(+), 16 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 99f91efe3824..006ac8ee9c7c 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -86,7 +86,9 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
+static dev_t nvme_ns_base_chr_devt;
 static struct class *nvme_class;
+static struct class *nvme_ns_class;
 static struct class *nvme_subsys_class;
 
 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
@@ -538,6 +540,7 @@ static void nvme_free_ns(struct kref *kref)
 	if (ns->ndev)
 		nvme_nvm_unregister(ns);
 
+	cdev_device_del(&ns->cdev, &ns->cdev_device);
 	put_disk(ns->disk);
 	nvme_put_ns_head(ns->head);
 	nvme_put_ctrl(ns->ctrl);
@@ -1738,15 +1741,15 @@ static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
 	return ret;
 }
 
-static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
-		unsigned int cmd, unsigned long arg)
+static int nvme_disk_ioctl(struct gendisk *disk, unsigned int cmd,
+			   unsigned long arg)
 {
 	struct nvme_ns_head *head = NULL;
 	void __user *argp = (void __user *)arg;
 	struct nvme_ns *ns;
 	int srcu_idx, ret;
 
-	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
+	ns = nvme_get_ns_from_disk(disk, &head, &srcu_idx);
 	if (unlikely(!ns))
 		return -EWOULDBLOCK;
 
@@ -1783,6 +1786,12 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
 	return ret;
 }
 
+static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
+		      unsigned int cmd, unsigned long arg)
+{
+	return nvme_disk_ioctl(bdev->bd_disk, cmd, arg);
+}
+
 #ifdef CONFIG_COMPAT
 struct nvme_user_io32 {
 	__u8	opcode;
@@ -1824,10 +1833,8 @@ static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
 #define nvme_compat_ioctl	NULL
 #endif /* CONFIG_COMPAT */
 
-static int nvme_open(struct block_device *bdev, fmode_t mode)
+static int nvme_ns_open(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = bdev->bd_disk->private_data;
-
 #ifdef CONFIG_NVME_MULTIPATH
 	/* should never be called due to GENHD_FL_HIDDEN */
 	if (WARN_ON_ONCE(ns->head->disk))
@@ -1846,14 +1853,22 @@ static int nvme_open(struct block_device *bdev, fmode_t mode)
 	return -ENXIO;
 }
 
-static void nvme_release(struct gendisk *disk, fmode_t mode)
+static void nvme_ns_release(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = disk->private_data;
-
 	module_put(ns->ctrl->ops->module);
 	nvme_put_ns(ns);
 }
 
+static int nvme_open(struct block_device *bdev, fmode_t mode)
+{
+	return nvme_ns_open(bdev->bd_disk->private_data);
+}
+
+static void nvme_release(struct gendisk *disk, fmode_t mode)
+{
+	nvme_ns_release(disk->private_data);
+}
+
 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 {
 	/* some standard values */
@@ -2209,6 +2224,13 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
 	return 0;
 
 out_unfreeze:
+	/*
+	 * When the device does not support any of the features required by the
+	 * kernel (or viceversa), hide the block device. We can still rely on
+	 * the namespace char device for submitting IOCTLs
+	 */
+	ns->disk->flags |= GENHD_FL_HIDDEN;
+
 	blk_mq_unfreeze_queue(ns->disk->queue);
 	return ret;
 }
@@ -2346,6 +2368,38 @@ static const struct block_device_operations nvme_bdev_ops = {
 	.pr_ops		= &nvme_pr_ops,
 };
 
+static int nvme_cdev_open(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	return nvme_ns_open(ns);
+}
+
+static int nvme_cdev_release(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	nvme_ns_release(ns);
+	return 0;
+}
+
+static long nvme_cdev_ioctl(struct file *file, unsigned int cmd,
+			    unsigned long arg)
+{
+	struct nvme_ns *ns = container_of(file->f_inode->i_cdev,
+				struct nvme_ns, cdev);
+
+	return nvme_disk_ioctl(ns->disk, cmd, arg);
+}
+
+static const struct file_operations nvme_cdev_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvme_cdev_open,
+	.release	= nvme_cdev_release,
+	.unlocked_ioctl	= nvme_cdev_ioctl,
+	.compat_ioctl	= compat_ptr_ioctl,
+};
+
 #ifdef CONFIG_NVME_MULTIPATH
 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
 {
@@ -3343,6 +3397,9 @@ static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
 {
 	struct gendisk *disk = dev_to_disk(dev);
 
+	if (dev->class == nvme_ns_class)
+		return nvme_get_ns_from_cdev(dev)->head;
+
 	if (disk->fops == &nvme_bdev_ops)
 		return nvme_get_ns_from_dev(dev)->head;
 	else
@@ -3474,6 +3531,11 @@ const struct attribute_group *nvme_ns_id_attr_groups[] = {
 	NULL,
 };
 
+const struct attribute_group *nvme_ns_char_id_attr_groups[] = {
+	&nvme_ns_id_attr_group,
+	NULL,
+};
+
 #define nvme_show_str_function(field)						\
 static ssize_t  field##_show(struct device *dev,				\
 			    struct device_attribute *attr, char *buf)		\
@@ -3866,6 +3928,36 @@ struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 }
 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
 
+static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
+{
+	char cdisk_name[DISK_NAME_LEN];
+	int ret;
+
+	device_initialize(&ns->cdev_device);
+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
+				     ns->head->instance);
+	ns->cdev_device.class = nvme_ns_class;
+	ns->cdev_device.parent = ctrl->device;
+	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
+	dev_set_drvdata(&ns->cdev_device, ns);
+
+	sprintf(cdisk_name, "nvme-generic-%dc%dn%d", ctrl->subsys->instance,
+		ctrl->instance, ns->head->instance);
+
+	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
+	if (ret)
+		return ret;
+
+	cdev_init(&ns->cdev, &nvme_cdev_fops);
+	ns->cdev.owner = ctrl->ops->module;
+
+	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
+	if (ret)
+		kfree_const(ns->cdev_device.kobj.name);
+
+	return ret;
+}
+
 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 		struct nvme_ns_ids *ids)
 {
@@ -3912,8 +4004,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
 	ns->disk = disk;
 
-	if (nvme_update_ns_info(ns, id))
-		goto out_put_disk;
+	nvme_update_ns_info(ns, id);
 
 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
 		if (nvme_nvm_register(ns, disk_name, node)) {
@@ -3929,9 +4020,12 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	nvme_get_ctrl(ctrl);
 
 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
-
 	nvme_mpath_add_disk(ns, id);
 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
+
+	if (nvme_alloc_chardev_ns(ctrl, ns))
+		goto out_put_disk;
+
 	kfree(id);
 
 	return;
@@ -4733,23 +4827,38 @@ static int __init nvme_core_init(void)
 	if (result < 0)
 		goto destroy_delete_wq;
 
+	result = alloc_chrdev_region(&nvme_ns_base_chr_devt, 0,
+			NVME_MINORS, "nvmec");
+	if (result < 0)
+		goto unregister_dev_chrdev;
+
 	nvme_class = class_create(THIS_MODULE, "nvme");
 	if (IS_ERR(nvme_class)) {
 		result = PTR_ERR(nvme_class);
-		goto unregister_chrdev;
+		goto unregister_ns_chrdev;
 	}
 	nvme_class->dev_uevent = nvme_class_uevent;
 
+	nvme_ns_class = class_create(THIS_MODULE, "nvme-ns");
+	if (IS_ERR(nvme_ns_class)) {
+		result = PTR_ERR(nvme_ns_class);
+		goto destroy_dev_class;
+	}
+
 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
 	if (IS_ERR(nvme_subsys_class)) {
 		result = PTR_ERR(nvme_subsys_class);
-		goto destroy_class;
+		goto destroy_ns_class;
 	}
 	return 0;
 
-destroy_class:
+destroy_ns_class:
+	class_destroy(nvme_ns_class);
+destroy_dev_class:
 	class_destroy(nvme_class);
-unregister_chrdev:
+unregister_ns_chrdev:
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
+unregister_dev_chrdev:
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 destroy_delete_wq:
 	destroy_workqueue(nvme_delete_wq);
@@ -4765,6 +4874,7 @@ static void __exit nvme_core_exit(void)
 {
 	class_destroy(nvme_subsys_class);
 	class_destroy(nvme_class);
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 	destroy_workqueue(nvme_delete_wq);
 	destroy_workqueue(nvme_reset_wq);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index bfcedfa4b057..1dd99f207aee 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -439,6 +439,9 @@ struct nvme_ns {
 	struct kref kref;
 	struct nvme_ns_head *head;
 
+	struct device cdev_device;	/* char device */
+	struct cdev cdev;
+
 	int lba_shift;
 	u16 ms;
 	u16 sgs;
@@ -818,6 +821,11 @@ static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
 	return dev_to_disk(dev)->private_data;
 }
 
+static inline struct nvme_ns *nvme_get_ns_from_cdev(struct device *dev)
+{
+	return dev_get_drvdata(dev);
+}
+
 #ifdef CONFIG_NVME_HWMON
 int nvme_hwmon_init(struct nvme_ctrl *ctrl);
 #else
-- 
2.17.1


_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: [PATCH V3] nvme: enable char device per namespace

[Keith Busch]

On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
> From: Javier González <javier.gonz@samsung.com>
> 
> Create a char device per NVMe namespace. This char device is always
> initialized, independently of whether the features implemented by the
> device are supported by the kernel. User-space can therefore always
> issue IOCTLs to the NVMe driver using the char device.
> 
> The char device is presented as /dev/generic-nvmeXcYnZ. This naming
> scheme follows the convention of the hidden device (nvmeXcYnZ). Support
> for multipath will follow.
> 
> Christoph, Keith: Is this going in the right direction?

I think this is looking okay, though I'm getting some weird errors and
warnings during boot. The first one looks like the following (I will
look into it too, but I wanted to get a reply out sooner).

[    4.734143] sysfs: cannot create duplicate filename '/dev/char/242:1'
[    4.736359] CPU: 4 PID: 7 Comm: kworker/u16:0 Not tainted 5.10.0+ #172
[    4.740836] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
[    4.744228] Workqueue: nvme-wq nvme_scan_work [nvme_core]
[    4.745558] Call Trace:
[    4.746122]  dump_stack+0x6d/0x88
[    4.746834]  sysfs_warn_dup.cold+0x17/0x24
[    4.747656]  sysfs_do_create_link_sd.isra.0+0xb0/0xc0
[    4.747659]  device_add+0x604/0x7b0
[    4.749298]  cdev_device_add+0x46/0x70
[    4.753241]  ? cdev_init+0x51/0x60
[    4.753246]  nvme_alloc_ns+0x670/0x8a0 [nvme_core]
[    4.753249]  ? _cond_resched+0x15/0x30
[    4.753253]  nvme_validate_or_alloc_ns+0x99/0x190 [nvme_core]
[    4.753258]  nvme_scan_work+0x152/0x290 [nvme_core]
[    4.753261]  process_one_work+0x1ac/0x330
[    4.753262]  worker_thread+0x50/0x3a0
[    4.753264]  ? process_one_work+0x330/0x330
[    4.753265]  kthread+0xfb/0x130
[    4.753266]  ? kthread_park+0x90/0x90
[    4.753269]  ret_from_fork+0x1f/0x30
[    4.753272] CPU: 5 PID: 88 Comm: kworker/u16:1 Not tainted 5.10.0+ #172

> Keith: Regarding nvme-cli support, what do you think about reporting the
> char device as existing block devices? If this is OK with you, I will
> submit patches for the filters.

I'm not sure I understand what you mean about reporting these as
"existing block devices". Are you talking about what's shown in the
'nvme list' output?

Re: [PATCH V3] nvme: enable char device per namespace

[Keith Busch]

On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
> From: Javier González <javier.gonz@samsung.com>
> 
> Create a char device per NVMe namespace. This char device is always
> initialized, independently of whether the features implemented by the
> device are supported by the kernel. User-space can therefore always
> issue IOCTLs to the NVMe driver using the char device.
> 
> The char device is presented as /dev/generic-nvmeXcYnZ. This naming
> scheme follows the convention of the hidden device (nvmeXcYnZ). Support
> for multipath will follow.
> 
> Christoph, Keith: Is this going in the right direction?

I think this is looking okay, though I'm getting some weird errors and
warnings during boot. The first one looks like the following (I will
look into it too, but I wanted to get a reply out sooner).

[    4.734143] sysfs: cannot create duplicate filename '/dev/char/242:1'
[    4.736359] CPU: 4 PID: 7 Comm: kworker/u16:0 Not tainted 5.10.0+ #172
[    4.740836] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
[    4.744228] Workqueue: nvme-wq nvme_scan_work [nvme_core]
[    4.745558] Call Trace:
[    4.746122]  dump_stack+0x6d/0x88
[    4.746834]  sysfs_warn_dup.cold+0x17/0x24
[    4.747656]  sysfs_do_create_link_sd.isra.0+0xb0/0xc0
[    4.747659]  device_add+0x604/0x7b0
[    4.749298]  cdev_device_add+0x46/0x70
[    4.753241]  ? cdev_init+0x51/0x60
[    4.753246]  nvme_alloc_ns+0x670/0x8a0 [nvme_core]
[    4.753249]  ? _cond_resched+0x15/0x30
[    4.753253]  nvme_validate_or_alloc_ns+0x99/0x190 [nvme_core]
[    4.753258]  nvme_scan_work+0x152/0x290 [nvme_core]
[    4.753261]  process_one_work+0x1ac/0x330
[    4.753262]  worker_thread+0x50/0x3a0
[    4.753264]  ? process_one_work+0x330/0x330
[    4.753265]  kthread+0xfb/0x130
[    4.753266]  ? kthread_park+0x90/0x90
[    4.753269]  ret_from_fork+0x1f/0x30
[    4.753272] CPU: 5 PID: 88 Comm: kworker/u16:1 Not tainted 5.10.0+ #172

> Keith: Regarding nvme-cli support, what do you think about reporting the
> char device as existing block devices? If this is OK with you, I will
> submit patches for the filters.

I'm not sure I understand what you mean about reporting these as
"existing block devices". Are you talking about what's shown in the
'nvme list' output?

_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: [PATCH V3] nvme: enable char device per namespace

[Keith Busch]

On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
> +{
> +	char cdisk_name[DISK_NAME_LEN];
> +	int ret;
> +
> +	device_initialize(&ns->cdev_device);
> +	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
> +				     ns->head->instance);

Ah, I see now. We are making these generic handles for each path, but
the ns->head->instance is the same for all paths to a namespace, so it's
not unique for that. Further, that head->instance is allocated per
subsystem, so it's not unique from namespace heads seen in other
subsystems.

So, I think you need to allocate a new dev_t for each subsystem rather
than the global nvme_ns_base_chr_devt, and I guess we also need a new
nvme_ns instance field assigned from yet another ida?

Re: [PATCH V3] nvme: enable char device per namespace

[Keith Busch]

On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
> +{
> +	char cdisk_name[DISK_NAME_LEN];
> +	int ret;
> +
> +	device_initialize(&ns->cdev_device);
> +	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
> +				     ns->head->instance);

Ah, I see now. We are making these generic handles for each path, but
the ns->head->instance is the same for all paths to a namespace, so it's
not unique for that. Further, that head->instance is allocated per
subsystem, so it's not unique from namespace heads seen in other
subsystems.

So, I think you need to allocate a new dev_t for each subsystem rather
than the global nvme_ns_base_chr_devt, and I guess we also need a new
nvme_ns instance field assigned from yet another ida?

_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: [PATCH V3] nvme: enable char device per namespace

[Javier González]

> On 15 Dec 2020, at 23.12, Keith Busch <kbusch@kernel.org> wrote:
> 
> On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
>> From: Javier González <javier.gonz@samsung.com>
>> 
>> Create a char device per NVMe namespace. This char device is always
>> initialized, independently of whether the features implemented by the
>> device are supported by the kernel. User-space can therefore always
>> issue IOCTLs to the NVMe driver using the char device.
>> 
>> The char device is presented as /dev/generic-nvmeXcYnZ. This naming
>> scheme follows the convention of the hidden device (nvmeXcYnZ). Support
>> for multipath will follow.
>> 
>> Christoph, Keith: Is this going in the right direction?
> 
> I think this is looking okay, though I'm getting some weird errors and
> warnings during boot. The first one looks like the following (I will
> look into it too, but I wanted to get a reply out sooner).
> 
> [    4.734143] sysfs: cannot create duplicate filename '/dev/char/242:1'
> [    4.736359] CPU: 4 PID: 7 Comm: kworker/u16:0 Not tainted 5.10.0+ #172
> [    4.740836] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
> [    4.744228] Workqueue: nvme-wq nvme_scan_work [nvme_core]
> [    4.745558] Call Trace:
> [    4.746122]  dump_stack+0x6d/0x88
> [    4.746834]  sysfs_warn_dup.cold+0x17/0x24
> [    4.747656]  sysfs_do_create_link_sd.isra.0+0xb0/0xc0
> [    4.747659]  device_add+0x604/0x7b0
> [    4.749298]  cdev_device_add+0x46/0x70
> [    4.753241]  ? cdev_init+0x51/0x60
> [    4.753246]  nvme_alloc_ns+0x670/0x8a0 [nvme_core]
> [    4.753249]  ? _cond_resched+0x15/0x30
> [    4.753253]  nvme_validate_or_alloc_ns+0x99/0x190 [nvme_core]
> [    4.753258]  nvme_scan_work+0x152/0x290 [nvme_core]
> [    4.753261]  process_one_work+0x1ac/0x330
> [    4.753262]  worker_thread+0x50/0x3a0
> [    4.753264]  ? process_one_work+0x330/0x330
> [    4.753265]  kthread+0xfb/0x130
> [    4.753266]  ? kthread_park+0x90/0x90
> [    4.753269]  ret_from_fork+0x1f/0x30
> [    4.753272] CPU: 5 PID: 88 Comm: kworker/u16:1 Not tainted 5.10.0+ #172

Mmm. I’ll look into it. I don’t see this in the config I’m using. 

> 
>> Keith: Regarding nvme-cli support, what do you think about reporting the
>> char device as existing block devices? If this is OK with you, I will
>> submit patches for the filters.
> 
> I'm not sure I understand what you mean about reporting these as
> "existing block devices". Are you talking about what's shown in the
> 'nvme list' output?

Exactly. Do we want the char device to be listed?

Re: [PATCH V3] nvme: enable char device per namespace

[Javier González]

> On 15 Dec 2020, at 23.12, Keith Busch <kbusch@kernel.org> wrote:
> 
> On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
>> From: Javier González <javier.gonz@samsung.com>
>> 
>> Create a char device per NVMe namespace. This char device is always
>> initialized, independently of whether the features implemented by the
>> device are supported by the kernel. User-space can therefore always
>> issue IOCTLs to the NVMe driver using the char device.
>> 
>> The char device is presented as /dev/generic-nvmeXcYnZ. This naming
>> scheme follows the convention of the hidden device (nvmeXcYnZ). Support
>> for multipath will follow.
>> 
>> Christoph, Keith: Is this going in the right direction?
> 
> I think this is looking okay, though I'm getting some weird errors and
> warnings during boot. The first one looks like the following (I will
> look into it too, but I wanted to get a reply out sooner).
> 
> [    4.734143] sysfs: cannot create duplicate filename '/dev/char/242:1'
> [    4.736359] CPU: 4 PID: 7 Comm: kworker/u16:0 Not tainted 5.10.0+ #172
> [    4.740836] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
> [    4.744228] Workqueue: nvme-wq nvme_scan_work [nvme_core]
> [    4.745558] Call Trace:
> [    4.746122]  dump_stack+0x6d/0x88
> [    4.746834]  sysfs_warn_dup.cold+0x17/0x24
> [    4.747656]  sysfs_do_create_link_sd.isra.0+0xb0/0xc0
> [    4.747659]  device_add+0x604/0x7b0
> [    4.749298]  cdev_device_add+0x46/0x70
> [    4.753241]  ? cdev_init+0x51/0x60
> [    4.753246]  nvme_alloc_ns+0x670/0x8a0 [nvme_core]
> [    4.753249]  ? _cond_resched+0x15/0x30
> [    4.753253]  nvme_validate_or_alloc_ns+0x99/0x190 [nvme_core]
> [    4.753258]  nvme_scan_work+0x152/0x290 [nvme_core]
> [    4.753261]  process_one_work+0x1ac/0x330
> [    4.753262]  worker_thread+0x50/0x3a0
> [    4.753264]  ? process_one_work+0x330/0x330
> [    4.753265]  kthread+0xfb/0x130
> [    4.753266]  ? kthread_park+0x90/0x90
> [    4.753269]  ret_from_fork+0x1f/0x30
> [    4.753272] CPU: 5 PID: 88 Comm: kworker/u16:1 Not tainted 5.10.0+ #172

Mmm. I’ll look into it. I don’t see this in the config I’m using. 

> 
>> Keith: Regarding nvme-cli support, what do you think about reporting the
>> char device as existing block devices? If this is OK with you, I will
>> submit patches for the filters.
> 
> I'm not sure I understand what you mean about reporting these as
> "existing block devices". Are you talking about what's shown in the
> 'nvme list' output?

Exactly. Do we want the char device to be listed?
_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: [PATCH V3] nvme: enable char device per namespace

[Javier González]

> On 15 Dec 2020, at 23.46, Keith Busch <kbusch@kernel.org> wrote:
> 
> On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
>> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
>> +{
>> +    char cdisk_name[DISK_NAME_LEN];
>> +    int ret;
>> +
>> +    device_initialize(&ns->cdev_device);
>> +    ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
>> +                     ns->head->instance);
> 
> Ah, I see now. We are making these generic handles for each path, but
> the ns->head->instance is the same for all paths to a namespace, so it's
> not unique for that. Further, that head->instance is allocated per
> subsystem, so it's not unique from namespace heads seen in other
> subsystems.
> 
> So, I think you need to allocate a new dev_t for each subsystem rather
> than the global nvme_ns_base_chr_devt, and I guess we also need a new
> nvme_ns instance field assigned from yet another ida?

Ok. I’ll look into it. 

Thanks!

Re: [PATCH V3] nvme: enable char device per namespace

[Javier González]

> On 15 Dec 2020, at 23.46, Keith Busch <kbusch@kernel.org> wrote:
> 
> On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
>> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
>> +{
>> +    char cdisk_name[DISK_NAME_LEN];
>> +    int ret;
>> +
>> +    device_initialize(&ns->cdev_device);
>> +    ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
>> +                     ns->head->instance);
> 
> Ah, I see now. We are making these generic handles for each path, but
> the ns->head->instance is the same for all paths to a namespace, so it's
> not unique for that. Further, that head->instance is allocated per
> subsystem, so it's not unique from namespace heads seen in other
> subsystems.
> 
> So, I think you need to allocate a new dev_t for each subsystem rather
> than the global nvme_ns_base_chr_devt, and I guess we also need a new
> nvme_ns instance field assigned from yet another ida?

Ok. I’ll look into it. 

Thanks!
_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: [PATCH V3] nvme: enable char device per namespace

[Keith Busch]

On Wed, Dec 16, 2020 at 09:01:51AM +0100, Javier González wrote:
> > On 15 Dec 2020, at 23.46, Keith Busch <kbusch@kernel.org> wrote:
> > On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
> >> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
> >> +{
> >> +    char cdisk_name[DISK_NAME_LEN];
> >> +    int ret;
> >> +
> >> +    device_initialize(&ns->cdev_device);
> >> +    ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
> >> +                     ns->head->instance);
> > 
> > Ah, I see now. We are making these generic handles for each path, but
> > the ns->head->instance is the same for all paths to a namespace, so it's
> > not unique for that. Further, that head->instance is allocated per
> > subsystem, so it's not unique from namespace heads seen in other
> > subsystems.
> > 
> > So, I think you need to allocate a new dev_t for each subsystem rather
> > than the global nvme_ns_base_chr_devt, and I guess we also need a new
> > nvme_ns instance field assigned from yet another ida?
> 
> Ok. I’ll look into it. 

The suggestion may be overkill as we don't need unique majors for each
controller right now (that may change if people need more than a
million generic handles, but I think we're a ways off from that reality).

The following on top of your patch makes it all work for me. Also, I
don't think we should abort adding the namespace if the generic handle
fails, so that's included here too:

---
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index c1aa4bccdeb2..cc9eaf4eba32 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -86,6 +86,8 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
+
+static DEFINE_IDA(nvme_gen_minor_ida);
 static dev_t nvme_ns_base_chr_devt;
 static struct class *nvme_class;
 static struct class *nvme_ns_class;
@@ -539,7 +541,8 @@ static void nvme_free_ns(struct kref *kref)
 
 	if (ns->ndev)
 		nvme_nvm_unregister(ns);
-
+	if (ns->minor)
+		ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
 	cdev_device_del(&ns->cdev, &ns->cdev_device);
 	put_disk(ns->disk);
 	nvme_put_ns_head(ns->head);
@@ -3932,9 +3935,13 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
 	char cdisk_name[DISK_NAME_LEN];
 	int ret;
 
+	ret = ida_simple_get(&nvme_gen_minor_ida, 0, 0, GFP_KERNEL);
+	if (ret < 0)
+		return ret;
+
+	ns->minor = ret + 1;
 	device_initialize(&ns->cdev_device);
-	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
-				     ns->head->instance);
+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt), ret);
 	ns->cdev_device.class = nvme_ns_class;
 	ns->cdev_device.parent = ctrl->device;
 	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
@@ -3945,15 +3952,22 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
 
 	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
 	if (ret)
-		return ret;
+		goto put_ida;
 
 	cdev_init(&ns->cdev, &nvme_cdev_fops);
 	ns->cdev.owner = ctrl->ops->module;
 
 	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
 	if (ret)
-		kfree_const(ns->cdev_device.kobj.name);
+		goto free_kobj;
+
+	return ret;
 
+free_kobj:
+	kfree_const(ns->cdev_device.kobj.name);
+put_ida:
+	ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
+	ns->minor = 0;
 	return ret;
 }
 
@@ -4023,7 +4037,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
 
 	if (nvme_alloc_chardev_ns(ctrl, ns))
-		goto out_put_disk;
+		dev_warn(ctrl->device,
+			"failed to create generic handle for nsid:%d\n",
+			nsid);
 
 	kfree(id);
 
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 168c7719cda4..ccfd49d2a030 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -435,6 +435,7 @@ struct nvme_ns {
 
 	struct device cdev_device;	/* char device */
 	struct cdev cdev;
+	int minor;
 
 	int lba_shift;
 	u16 ms;
--

Re: [PATCH V3] nvme: enable char device per namespace

[Keith Busch]

On Wed, Dec 16, 2020 at 09:01:51AM +0100, Javier González wrote:
> > On 15 Dec 2020, at 23.46, Keith Busch <kbusch@kernel.org> wrote:
> > On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
> >> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
> >> +{
> >> +    char cdisk_name[DISK_NAME_LEN];
> >> +    int ret;
> >> +
> >> +    device_initialize(&ns->cdev_device);
> >> +    ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
> >> +                     ns->head->instance);
> > 
> > Ah, I see now. We are making these generic handles for each path, but
> > the ns->head->instance is the same for all paths to a namespace, so it's
> > not unique for that. Further, that head->instance is allocated per
> > subsystem, so it's not unique from namespace heads seen in other
> > subsystems.
> > 
> > So, I think you need to allocate a new dev_t for each subsystem rather
> > than the global nvme_ns_base_chr_devt, and I guess we also need a new
> > nvme_ns instance field assigned from yet another ida?
> 
> Ok. I’ll look into it. 

The suggestion may be overkill as we don't need unique majors for each
controller right now (that may change if people need more than a
million generic handles, but I think we're a ways off from that reality).

The following on top of your patch makes it all work for me. Also, I
don't think we should abort adding the namespace if the generic handle
fails, so that's included here too:

---
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index c1aa4bccdeb2..cc9eaf4eba32 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -86,6 +86,8 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
+
+static DEFINE_IDA(nvme_gen_minor_ida);
 static dev_t nvme_ns_base_chr_devt;
 static struct class *nvme_class;
 static struct class *nvme_ns_class;
@@ -539,7 +541,8 @@ static void nvme_free_ns(struct kref *kref)
 
 	if (ns->ndev)
 		nvme_nvm_unregister(ns);
-
+	if (ns->minor)
+		ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
 	cdev_device_del(&ns->cdev, &ns->cdev_device);
 	put_disk(ns->disk);
 	nvme_put_ns_head(ns->head);
@@ -3932,9 +3935,13 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
 	char cdisk_name[DISK_NAME_LEN];
 	int ret;
 
+	ret = ida_simple_get(&nvme_gen_minor_ida, 0, 0, GFP_KERNEL);
+	if (ret < 0)
+		return ret;
+
+	ns->minor = ret + 1;
 	device_initialize(&ns->cdev_device);
-	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
-				     ns->head->instance);
+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt), ret);
 	ns->cdev_device.class = nvme_ns_class;
 	ns->cdev_device.parent = ctrl->device;
 	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
@@ -3945,15 +3952,22 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
 
 	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
 	if (ret)
-		return ret;
+		goto put_ida;
 
 	cdev_init(&ns->cdev, &nvme_cdev_fops);
 	ns->cdev.owner = ctrl->ops->module;
 
 	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
 	if (ret)
-		kfree_const(ns->cdev_device.kobj.name);
+		goto free_kobj;
+
+	return ret;
 
+free_kobj:
+	kfree_const(ns->cdev_device.kobj.name);
+put_ida:
+	ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
+	ns->minor = 0;
 	return ret;
 }
 
@@ -4023,7 +4037,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
 
 	if (nvme_alloc_chardev_ns(ctrl, ns))
-		goto out_put_disk;
+		dev_warn(ctrl->device,
+			"failed to create generic handle for nsid:%d\n",
+			nsid);
 
 	kfree(id);
 
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 168c7719cda4..ccfd49d2a030 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -435,6 +435,7 @@ struct nvme_ns {
 
 	struct device cdev_device;	/* char device */
 	struct cdev cdev;
+	int minor;
 
 	int lba_shift;
 	u16 ms;
--

_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: nvme: enable char device per namespace

[Javier González]

On 16.12.2020 08:26, Keith Busch wrote:
>On Wed, Dec 16, 2020 at 09:01:51AM +0100, Javier González wrote:
>> > On 15 Dec 2020, at 23.46, Keith Busch <kbusch@kernel.org> wrote:
>> > On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
>> >> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
>> >> +{
>> >> +    char cdisk_name[DISK_NAME_LEN];
>> >> +    int ret;
>> >> +
>> >> +    device_initialize(&ns->cdev_device);
>> >> +    ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
>> >> +                     ns->head->instance);
>> >
>> > Ah, I see now. We are making these generic handles for each path, but
>> > the ns->head->instance is the same for all paths to a namespace, so it's
>> > not unique for that. Further, that head->instance is allocated per
>> > subsystem, so it's not unique from namespace heads seen in other
>> > subsystems.
>> >
>> > So, I think you need to allocate a new dev_t for each subsystem rather
>> > than the global nvme_ns_base_chr_devt, and I guess we also need a new
>> > nvme_ns instance field assigned from yet another ida?
>>
>> Ok. I’ll look into it.
>
>The suggestion may be overkill as we don't need unique majors for each
>controller right now (that may change if people need more than a
>million generic handles, but I think we're a ways off from that reality).
>
>The following on top of your patch makes it all work for me. Also, I
>don't think we should abort adding the namespace if the generic handle
>fails, so that's included here too:
>
>---
>diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
>index c1aa4bccdeb2..cc9eaf4eba32 100644
>--- a/drivers/nvme/host/core.c
>+++ b/drivers/nvme/host/core.c
>@@ -86,6 +86,8 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
>
> static DEFINE_IDA(nvme_instance_ida);
> static dev_t nvme_ctrl_base_chr_devt;
>+
>+static DEFINE_IDA(nvme_gen_minor_ida);
> static dev_t nvme_ns_base_chr_devt;
> static struct class *nvme_class;
> static struct class *nvme_ns_class;
>@@ -539,7 +541,8 @@ static void nvme_free_ns(struct kref *kref)
>
> 	if (ns->ndev)
> 		nvme_nvm_unregister(ns);
>-
>+	if (ns->minor)
>+		ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
> 	cdev_device_del(&ns->cdev, &ns->cdev_device);
> 	put_disk(ns->disk);
> 	nvme_put_ns_head(ns->head);
>@@ -3932,9 +3935,13 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
> 	char cdisk_name[DISK_NAME_LEN];
> 	int ret;
>
>+	ret = ida_simple_get(&nvme_gen_minor_ida, 0, 0, GFP_KERNEL);
>+	if (ret < 0)
>+		return ret;
>+
>+	ns->minor = ret + 1;
> 	device_initialize(&ns->cdev_device);
>-	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
>-				     ns->head->instance);
>+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt), ret);
> 	ns->cdev_device.class = nvme_ns_class;
> 	ns->cdev_device.parent = ctrl->device;
> 	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
>@@ -3945,15 +3952,22 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
>
> 	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
> 	if (ret)
>-		return ret;
>+		goto put_ida;
>
> 	cdev_init(&ns->cdev, &nvme_cdev_fops);
> 	ns->cdev.owner = ctrl->ops->module;
>
> 	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
> 	if (ret)
>-		kfree_const(ns->cdev_device.kobj.name);
>+		goto free_kobj;
>+
>+	return ret;
>
>+free_kobj:
>+	kfree_const(ns->cdev_device.kobj.name);
>+put_ida:
>+	ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
>+	ns->minor = 0;
> 	return ret;
> }
>
>@@ -4023,7 +4037,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
> 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
>
> 	if (nvme_alloc_chardev_ns(ctrl, ns))
>-		goto out_put_disk;
>+		dev_warn(ctrl->device,
>+			"failed to create generic handle for nsid:%d\n",
>+			nsid);
>
> 	kfree(id);
>
>diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
>index 168c7719cda4..ccfd49d2a030 100644
>--- a/drivers/nvme/host/nvme.h
>+++ b/drivers/nvme/host/nvme.h
>@@ -435,6 +435,7 @@ struct nvme_ns {
>
> 	struct device cdev_device;	/* char device */
> 	struct cdev cdev;
>+	int minor;
>
> 	int lba_shift;
> 	u16 ms;
>--

Thanks Keith. I will send a new version today.

Regarding nvme-cli: what are your thoughts?

Re: nvme: enable char device per namespace

[Javier González]

On 16.12.2020 08:26, Keith Busch wrote:
>On Wed, Dec 16, 2020 at 09:01:51AM +0100, Javier González wrote:
>> > On 15 Dec 2020, at 23.46, Keith Busch <kbusch@kernel.org> wrote:
>> > On Tue, Dec 15, 2020 at 08:55:57PM +0100, javier@javigon.com wrote:
>> >> +static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
>> >> +{
>> >> +    char cdisk_name[DISK_NAME_LEN];
>> >> +    int ret;
>> >> +
>> >> +    device_initialize(&ns->cdev_device);
>> >> +    ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
>> >> +                     ns->head->instance);
>> >
>> > Ah, I see now. We are making these generic handles for each path, but
>> > the ns->head->instance is the same for all paths to a namespace, so it's
>> > not unique for that. Further, that head->instance is allocated per
>> > subsystem, so it's not unique from namespace heads seen in other
>> > subsystems.
>> >
>> > So, I think you need to allocate a new dev_t for each subsystem rather
>> > than the global nvme_ns_base_chr_devt, and I guess we also need a new
>> > nvme_ns instance field assigned from yet another ida?
>>
>> Ok. I’ll look into it.
>
>The suggestion may be overkill as we don't need unique majors for each
>controller right now (that may change if people need more than a
>million generic handles, but I think we're a ways off from that reality).
>
>The following on top of your patch makes it all work for me. Also, I
>don't think we should abort adding the namespace if the generic handle
>fails, so that's included here too:
>
>---
>diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
>index c1aa4bccdeb2..cc9eaf4eba32 100644
>--- a/drivers/nvme/host/core.c
>+++ b/drivers/nvme/host/core.c
>@@ -86,6 +86,8 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
>
> static DEFINE_IDA(nvme_instance_ida);
> static dev_t nvme_ctrl_base_chr_devt;
>+
>+static DEFINE_IDA(nvme_gen_minor_ida);
> static dev_t nvme_ns_base_chr_devt;
> static struct class *nvme_class;
> static struct class *nvme_ns_class;
>@@ -539,7 +541,8 @@ static void nvme_free_ns(struct kref *kref)
>
> 	if (ns->ndev)
> 		nvme_nvm_unregister(ns);
>-
>+	if (ns->minor)
>+		ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
> 	cdev_device_del(&ns->cdev, &ns->cdev_device);
> 	put_disk(ns->disk);
> 	nvme_put_ns_head(ns->head);
>@@ -3932,9 +3935,13 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
> 	char cdisk_name[DISK_NAME_LEN];
> 	int ret;
>
>+	ret = ida_simple_get(&nvme_gen_minor_ida, 0, 0, GFP_KERNEL);
>+	if (ret < 0)
>+		return ret;
>+
>+	ns->minor = ret + 1;
> 	device_initialize(&ns->cdev_device);
>-	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
>-				     ns->head->instance);
>+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt), ret);
> 	ns->cdev_device.class = nvme_ns_class;
> 	ns->cdev_device.parent = ctrl->device;
> 	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
>@@ -3945,15 +3952,22 @@ static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
>
> 	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
> 	if (ret)
>-		return ret;
>+		goto put_ida;
>
> 	cdev_init(&ns->cdev, &nvme_cdev_fops);
> 	ns->cdev.owner = ctrl->ops->module;
>
> 	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
> 	if (ret)
>-		kfree_const(ns->cdev_device.kobj.name);
>+		goto free_kobj;
>+
>+	return ret;
>
>+free_kobj:
>+	kfree_const(ns->cdev_device.kobj.name);
>+put_ida:
>+	ida_simple_remove(&nvme_gen_minor_ida, ns->minor - 1);
>+	ns->minor = 0;
> 	return ret;
> }
>
>@@ -4023,7 +4037,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
> 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
>
> 	if (nvme_alloc_chardev_ns(ctrl, ns))
>-		goto out_put_disk;
>+		dev_warn(ctrl->device,
>+			"failed to create generic handle for nsid:%d\n",
>+			nsid);
>
> 	kfree(id);
>
>diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
>index 168c7719cda4..ccfd49d2a030 100644
>--- a/drivers/nvme/host/nvme.h
>+++ b/drivers/nvme/host/nvme.h
>@@ -435,6 +435,7 @@ struct nvme_ns {
>
> 	struct device cdev_device;	/* char device */
> 	struct cdev cdev;
>+	int minor;
>
> 	int lba_shift;
> 	u16 ms;
>--

Thanks Keith. I will send a new version today.

Regarding nvme-cli: what are your thoughts?


_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: nvme: enable char device per namespace

[Keith Busch]

On Wed, Dec 16, 2020 at 06:43:22PM +0100, Javier González wrote:
> Thanks Keith. I will send a new version today.
> 
> Regarding nvme-cli: what are your thoughts?

I was thinking we could add a column for these with the '--verbose'
option in the namespace section.

Re: nvme: enable char device per namespace

[Keith Busch]

On Wed, Dec 16, 2020 at 06:43:22PM +0100, Javier González wrote:
> Thanks Keith. I will send a new version today.
> 
> Regarding nvme-cli: what are your thoughts?

I was thinking we could add a column for these with the '--verbose'
option in the namespace section.

_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

Re: nvme: enable char device per namespace

[Javier González]

On 17.12.2020 02:53, Keith Busch wrote:
>On Wed, Dec 16, 2020 at 06:43:22PM +0100, Javier González wrote:
>> Thanks Keith. I will send a new version today.
>>
>> Regarding nvme-cli: what are your thoughts?
>
>I was thinking we could add a column for these with the '--verbose'
>option in the namespace section.

Makes sense. I will look into it - probably on the other side of
Christmas. Should give plenty of time since this till not be merged
until 5.12.

Should give me time to add multipath in a follow-up patch then.

Re: nvme: enable char device per namespace

[Javier González]

On 17.12.2020 02:53, Keith Busch wrote:
>On Wed, Dec 16, 2020 at 06:43:22PM +0100, Javier González wrote:
>> Thanks Keith. I will send a new version today.
>>
>> Regarding nvme-cli: what are your thoughts?
>
>I was thinking we could add a column for these with the '--verbose'
>option in the namespace section.

Makes sense. I will look into it - probably on the other side of
Christmas. Should give plenty of time since this till not be merged
until 5.12.

Should give me time to add multipath in a follow-up patch then.


_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

[PATCH V3] nvme: enable char device per namespace

[javier]

From: Javier González <javier.gonz@samsung.com>

Create a char device per NVMe namespace. This char device is always
initialized, independently of whether the features implemented by the
device are supported by the kernel. User-space can therefore always
issue IOCTLs to the NVMe driver using the char device.

The char device is presented as /dev/generic-nvmeXcYnZ. This naming
scheme follows the convention of the hidden device (nvmeXcYnZ). Support
for multipath will follow.

Keith: Regarding nvme-cli support, what do you think about reporting the
char device as existing block devices? If this is OK with you, I will
submit patches for the filters.

Changes since V2:
  - Apply a number of naming and code structure improvements (from
    Christoph)
  - Use i_cdev to pull struct nvme_ns in the ioctl path instead of
    populating file->private_data (from Christoph)
  - Change char device and sysfs entries to /dev/generic-nvmeXcYnZ to
    follow the hidden device naming scheme (from Christoph and Keith)

Changes since V1:
  - Remove patches 1-3 which are already picked up by Christoph
  - Change the char device and sysfs entries to nvmeXnYc / c signals
    char device
  - Address Minwoo's comments on inline functions and style

Signed-off-by: Javier González <javier.gonz@samsung.com>
---
 drivers/nvme/host/core.c | 142 ++++++++++++++++++++++++++++++++++-----
 drivers/nvme/host/nvme.h |   8 +++
 2 files changed, 134 insertions(+), 16 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 99f91efe3824..006ac8ee9c7c 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -86,7 +86,9 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
+static dev_t nvme_ns_base_chr_devt;
 static struct class *nvme_class;
+static struct class *nvme_ns_class;
 static struct class *nvme_subsys_class;
 
 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
@@ -538,6 +540,7 @@ static void nvme_free_ns(struct kref *kref)
 	if (ns->ndev)
 		nvme_nvm_unregister(ns);
 
+	cdev_device_del(&ns->cdev, &ns->cdev_device);
 	put_disk(ns->disk);
 	nvme_put_ns_head(ns->head);
 	nvme_put_ctrl(ns->ctrl);
@@ -1738,15 +1741,15 @@ static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
 	return ret;
 }
 
-static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
-		unsigned int cmd, unsigned long arg)
+static int nvme_disk_ioctl(struct gendisk *disk, unsigned int cmd,
+			   unsigned long arg)
 {
 	struct nvme_ns_head *head = NULL;
 	void __user *argp = (void __user *)arg;
 	struct nvme_ns *ns;
 	int srcu_idx, ret;
 
-	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
+	ns = nvme_get_ns_from_disk(disk, &head, &srcu_idx);
 	if (unlikely(!ns))
 		return -EWOULDBLOCK;
 
@@ -1783,6 +1786,12 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
 	return ret;
 }
 
+static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
+		      unsigned int cmd, unsigned long arg)
+{
+	return nvme_disk_ioctl(bdev->bd_disk, cmd, arg);
+}
+
 #ifdef CONFIG_COMPAT
 struct nvme_user_io32 {
 	__u8	opcode;
@@ -1824,10 +1833,8 @@ static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
 #define nvme_compat_ioctl	NULL
 #endif /* CONFIG_COMPAT */
 
-static int nvme_open(struct block_device *bdev, fmode_t mode)
+static int nvme_ns_open(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = bdev->bd_disk->private_data;
-
 #ifdef CONFIG_NVME_MULTIPATH
 	/* should never be called due to GENHD_FL_HIDDEN */
 	if (WARN_ON_ONCE(ns->head->disk))
@@ -1846,14 +1853,22 @@ static int nvme_open(struct block_device *bdev, fmode_t mode)
 	return -ENXIO;
 }
 
-static void nvme_release(struct gendisk *disk, fmode_t mode)
+static void nvme_ns_release(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = disk->private_data;
-
 	module_put(ns->ctrl->ops->module);
 	nvme_put_ns(ns);
 }
 
+static int nvme_open(struct block_device *bdev, fmode_t mode)
+{
+	return nvme_ns_open(bdev->bd_disk->private_data);
+}
+
+static void nvme_release(struct gendisk *disk, fmode_t mode)
+{
+	nvme_ns_release(disk->private_data);
+}
+
 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 {
 	/* some standard values */
@@ -2209,6 +2224,13 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
 	return 0;
 
 out_unfreeze:
+	/*
+	 * When the device does not support any of the features required by the
+	 * kernel (or viceversa), hide the block device. We can still rely on
+	 * the namespace char device for submitting IOCTLs
+	 */
+	ns->disk->flags |= GENHD_FL_HIDDEN;
+
 	blk_mq_unfreeze_queue(ns->disk->queue);
 	return ret;
 }
@@ -2346,6 +2368,38 @@ static const struct block_device_operations nvme_bdev_ops = {
 	.pr_ops		= &nvme_pr_ops,
 };
 
+static int nvme_cdev_open(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	return nvme_ns_open(ns);
+}
+
+static int nvme_cdev_release(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	nvme_ns_release(ns);
+	return 0;
+}
+
+static long nvme_cdev_ioctl(struct file *file, unsigned int cmd,
+			    unsigned long arg)
+{
+	struct nvme_ns *ns = container_of(file->f_inode->i_cdev,
+				struct nvme_ns, cdev);
+
+	return nvme_disk_ioctl(ns->disk, cmd, arg);
+}
+
+static const struct file_operations nvme_cdev_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvme_cdev_open,
+	.release	= nvme_cdev_release,
+	.unlocked_ioctl	= nvme_cdev_ioctl,
+	.compat_ioctl	= compat_ptr_ioctl,
+};
+
 #ifdef CONFIG_NVME_MULTIPATH
 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
 {
@@ -3343,6 +3397,9 @@ static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
 {
 	struct gendisk *disk = dev_to_disk(dev);
 
+	if (dev->class == nvme_ns_class)
+		return nvme_get_ns_from_cdev(dev)->head;
+
 	if (disk->fops == &nvme_bdev_ops)
 		return nvme_get_ns_from_dev(dev)->head;
 	else
@@ -3474,6 +3531,11 @@ const struct attribute_group *nvme_ns_id_attr_groups[] = {
 	NULL,
 };
 
+const struct attribute_group *nvme_ns_char_id_attr_groups[] = {
+	&nvme_ns_id_attr_group,
+	NULL,
+};
+
 #define nvme_show_str_function(field)						\
 static ssize_t  field##_show(struct device *dev,				\
 			    struct device_attribute *attr, char *buf)		\
@@ -3866,6 +3928,36 @@ struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 }
 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
 
+static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
+{
+	char cdisk_name[DISK_NAME_LEN];
+	int ret;
+
+	device_initialize(&ns->cdev_device);
+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
+				     ns->head->instance);
+	ns->cdev_device.class = nvme_ns_class;
+	ns->cdev_device.parent = ctrl->device;
+	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
+	dev_set_drvdata(&ns->cdev_device, ns);
+
+	sprintf(cdisk_name, "nvme-generic-%dc%dn%d", ctrl->subsys->instance,
+		ctrl->instance, ns->head->instance);
+
+	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
+	if (ret)
+		return ret;
+
+	cdev_init(&ns->cdev, &nvme_cdev_fops);
+	ns->cdev.owner = ctrl->ops->module;
+
+	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
+	if (ret)
+		kfree_const(ns->cdev_device.kobj.name);
+
+	return ret;
+}
+
 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 		struct nvme_ns_ids *ids)
 {
@@ -3912,8 +4004,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
 	ns->disk = disk;
 
-	if (nvme_update_ns_info(ns, id))
-		goto out_put_disk;
+	nvme_update_ns_info(ns, id);
 
 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
 		if (nvme_nvm_register(ns, disk_name, node)) {
@@ -3929,9 +4020,12 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	nvme_get_ctrl(ctrl);
 
 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
-
 	nvme_mpath_add_disk(ns, id);
 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
+
+	if (nvme_alloc_chardev_ns(ctrl, ns))
+		goto out_put_disk;
+
 	kfree(id);
 
 	return;
@@ -4733,23 +4827,38 @@ static int __init nvme_core_init(void)
 	if (result < 0)
 		goto destroy_delete_wq;
 
+	result = alloc_chrdev_region(&nvme_ns_base_chr_devt, 0,
+			NVME_MINORS, "nvmec");
+	if (result < 0)
+		goto unregister_dev_chrdev;
+
 	nvme_class = class_create(THIS_MODULE, "nvme");
 	if (IS_ERR(nvme_class)) {
 		result = PTR_ERR(nvme_class);
-		goto unregister_chrdev;
+		goto unregister_ns_chrdev;
 	}
 	nvme_class->dev_uevent = nvme_class_uevent;
 
+	nvme_ns_class = class_create(THIS_MODULE, "nvme-ns");
+	if (IS_ERR(nvme_ns_class)) {
+		result = PTR_ERR(nvme_ns_class);
+		goto destroy_dev_class;
+	}
+
 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
 	if (IS_ERR(nvme_subsys_class)) {
 		result = PTR_ERR(nvme_subsys_class);
-		goto destroy_class;
+		goto destroy_ns_class;
 	}
 	return 0;
 
-destroy_class:
+destroy_ns_class:
+	class_destroy(nvme_ns_class);
+destroy_dev_class:
 	class_destroy(nvme_class);
-unregister_chrdev:
+unregister_ns_chrdev:
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
+unregister_dev_chrdev:
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 destroy_delete_wq:
 	destroy_workqueue(nvme_delete_wq);
@@ -4765,6 +4874,7 @@ static void __exit nvme_core_exit(void)
 {
 	class_destroy(nvme_subsys_class);
 	class_destroy(nvme_class);
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 	destroy_workqueue(nvme_delete_wq);
 	destroy_workqueue(nvme_reset_wq);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index bfcedfa4b057..1dd99f207aee 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -439,6 +439,9 @@ struct nvme_ns {
 	struct kref kref;
 	struct nvme_ns_head *head;
 
+	struct device cdev_device;	/* char device */
+	struct cdev cdev;
+
 	int lba_shift;
 	u16 ms;
 	u16 sgs;
@@ -818,6 +821,11 @@ static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
 	return dev_to_disk(dev)->private_data;
 }
 
+static inline struct nvme_ns *nvme_get_ns_from_cdev(struct device *dev)
+{
+	return dev_get_drvdata(dev);
+}
+
 #ifdef CONFIG_NVME_HWMON
 int nvme_hwmon_init(struct nvme_ctrl *ctrl);
 #else
-- 
2.17.1

[PATCH V3] nvme: enable char device per namespace

[javier]

From: Javier González <javier.gonz@samsung.com>

Create a char device per NVMe namespace. This char device is always
initialized, independently of whether the features implemented by the
device are supported by the kernel. User-space can therefore always
issue IOCTLs to the NVMe driver using the char device.

The char device is presented as /dev/generic-nvmeXcYnZ. This naming
scheme follows the convention of the hidden device (nvmeXcYnZ). Support
for multipath will follow.

Keith: Regarding nvme-cli support, what do you think about reporting the
char device as existing block devices? If this is OK with you, I will
submit patches for the filters.

Changes since V2:
  - Apply a number of naming and code structure improvements (from
    Christoph)
  - Use i_cdev to pull struct nvme_ns in the ioctl path instead of
    populating file->private_data (from Christoph)
  - Change char device and sysfs entries to /dev/generic-nvmeXcYnZ to
    follow the hidden device naming scheme (from Christoph and Keith)

Changes since V1:
  - Remove patches 1-3 which are already picked up by Christoph
  - Change the char device and sysfs entries to nvmeXnYc / c signals
    char device
  - Address Minwoo's comments on inline functions and style

Signed-off-by: Javier González <javier.gonz@samsung.com>
---
 drivers/nvme/host/core.c | 142 ++++++++++++++++++++++++++++++++++-----
 drivers/nvme/host/nvme.h |   8 +++
 2 files changed, 134 insertions(+), 16 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 99f91efe3824..006ac8ee9c7c 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -86,7 +86,9 @@ static DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
+static dev_t nvme_ns_base_chr_devt;
 static struct class *nvme_class;
+static struct class *nvme_ns_class;
 static struct class *nvme_subsys_class;
 
 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
@@ -538,6 +540,7 @@ static void nvme_free_ns(struct kref *kref)
 	if (ns->ndev)
 		nvme_nvm_unregister(ns);
 
+	cdev_device_del(&ns->cdev, &ns->cdev_device);
 	put_disk(ns->disk);
 	nvme_put_ns_head(ns->head);
 	nvme_put_ctrl(ns->ctrl);
@@ -1738,15 +1741,15 @@ static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
 	return ret;
 }
 
-static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
-		unsigned int cmd, unsigned long arg)
+static int nvme_disk_ioctl(struct gendisk *disk, unsigned int cmd,
+			   unsigned long arg)
 {
 	struct nvme_ns_head *head = NULL;
 	void __user *argp = (void __user *)arg;
 	struct nvme_ns *ns;
 	int srcu_idx, ret;
 
-	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
+	ns = nvme_get_ns_from_disk(disk, &head, &srcu_idx);
 	if (unlikely(!ns))
 		return -EWOULDBLOCK;
 
@@ -1783,6 +1786,12 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
 	return ret;
 }
 
+static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
+		      unsigned int cmd, unsigned long arg)
+{
+	return nvme_disk_ioctl(bdev->bd_disk, cmd, arg);
+}
+
 #ifdef CONFIG_COMPAT
 struct nvme_user_io32 {
 	__u8	opcode;
@@ -1824,10 +1833,8 @@ static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
 #define nvme_compat_ioctl	NULL
 #endif /* CONFIG_COMPAT */
 
-static int nvme_open(struct block_device *bdev, fmode_t mode)
+static int nvme_ns_open(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = bdev->bd_disk->private_data;
-
 #ifdef CONFIG_NVME_MULTIPATH
 	/* should never be called due to GENHD_FL_HIDDEN */
 	if (WARN_ON_ONCE(ns->head->disk))
@@ -1846,14 +1853,22 @@ static int nvme_open(struct block_device *bdev, fmode_t mode)
 	return -ENXIO;
 }
 
-static void nvme_release(struct gendisk *disk, fmode_t mode)
+static void nvme_ns_release(struct nvme_ns *ns)
 {
-	struct nvme_ns *ns = disk->private_data;
-
 	module_put(ns->ctrl->ops->module);
 	nvme_put_ns(ns);
 }
 
+static int nvme_open(struct block_device *bdev, fmode_t mode)
+{
+	return nvme_ns_open(bdev->bd_disk->private_data);
+}
+
+static void nvme_release(struct gendisk *disk, fmode_t mode)
+{
+	nvme_ns_release(disk->private_data);
+}
+
 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 {
 	/* some standard values */
@@ -2209,6 +2224,13 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
 	return 0;
 
 out_unfreeze:
+	/*
+	 * When the device does not support any of the features required by the
+	 * kernel (or viceversa), hide the block device. We can still rely on
+	 * the namespace char device for submitting IOCTLs
+	 */
+	ns->disk->flags |= GENHD_FL_HIDDEN;
+
 	blk_mq_unfreeze_queue(ns->disk->queue);
 	return ret;
 }
@@ -2346,6 +2368,38 @@ static const struct block_device_operations nvme_bdev_ops = {
 	.pr_ops		= &nvme_pr_ops,
 };
 
+static int nvme_cdev_open(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	return nvme_ns_open(ns);
+}
+
+static int nvme_cdev_release(struct inode *inode, struct file *file)
+{
+	struct nvme_ns *ns = container_of(inode->i_cdev, struct nvme_ns, cdev);
+
+	nvme_ns_release(ns);
+	return 0;
+}
+
+static long nvme_cdev_ioctl(struct file *file, unsigned int cmd,
+			    unsigned long arg)
+{
+	struct nvme_ns *ns = container_of(file->f_inode->i_cdev,
+				struct nvme_ns, cdev);
+
+	return nvme_disk_ioctl(ns->disk, cmd, arg);
+}
+
+static const struct file_operations nvme_cdev_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvme_cdev_open,
+	.release	= nvme_cdev_release,
+	.unlocked_ioctl	= nvme_cdev_ioctl,
+	.compat_ioctl	= compat_ptr_ioctl,
+};
+
 #ifdef CONFIG_NVME_MULTIPATH
 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
 {
@@ -3343,6 +3397,9 @@ static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
 {
 	struct gendisk *disk = dev_to_disk(dev);
 
+	if (dev->class == nvme_ns_class)
+		return nvme_get_ns_from_cdev(dev)->head;
+
 	if (disk->fops == &nvme_bdev_ops)
 		return nvme_get_ns_from_dev(dev)->head;
 	else
@@ -3474,6 +3531,11 @@ const struct attribute_group *nvme_ns_id_attr_groups[] = {
 	NULL,
 };
 
+const struct attribute_group *nvme_ns_char_id_attr_groups[] = {
+	&nvme_ns_id_attr_group,
+	NULL,
+};
+
 #define nvme_show_str_function(field)						\
 static ssize_t  field##_show(struct device *dev,				\
 			    struct device_attribute *attr, char *buf)		\
@@ -3866,6 +3928,36 @@ struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 }
 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
 
+static int nvme_alloc_chardev_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
+{
+	char cdisk_name[DISK_NAME_LEN];
+	int ret;
+
+	device_initialize(&ns->cdev_device);
+	ns->cdev_device.devt = MKDEV(MAJOR(nvme_ns_base_chr_devt),
+				     ns->head->instance);
+	ns->cdev_device.class = nvme_ns_class;
+	ns->cdev_device.parent = ctrl->device;
+	ns->cdev_device.groups = nvme_ns_char_id_attr_groups;
+	dev_set_drvdata(&ns->cdev_device, ns);
+
+	sprintf(cdisk_name, "nvme-generic-%dc%dn%d", ctrl->subsys->instance,
+		ctrl->instance, ns->head->instance);
+
+	ret = dev_set_name(&ns->cdev_device, "%s", cdisk_name);
+	if (ret)
+		return ret;
+
+	cdev_init(&ns->cdev, &nvme_cdev_fops);
+	ns->cdev.owner = ctrl->ops->module;
+
+	ret = cdev_device_add(&ns->cdev, &ns->cdev_device);
+	if (ret)
+		kfree_const(ns->cdev_device.kobj.name);
+
+	return ret;
+}
+
 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 		struct nvme_ns_ids *ids)
 {
@@ -3912,8 +4004,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
 	ns->disk = disk;
 
-	if (nvme_update_ns_info(ns, id))
-		goto out_put_disk;
+	nvme_update_ns_info(ns, id);
 
 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
 		if (nvme_nvm_register(ns, disk_name, node)) {
@@ -3929,9 +4020,12 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
 	nvme_get_ctrl(ctrl);
 
 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
-
 	nvme_mpath_add_disk(ns, id);
 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
+
+	if (nvme_alloc_chardev_ns(ctrl, ns))
+		goto out_put_disk;
+
 	kfree(id);
 
 	return;
@@ -4733,23 +4827,38 @@ static int __init nvme_core_init(void)
 	if (result < 0)
 		goto destroy_delete_wq;
 
+	result = alloc_chrdev_region(&nvme_ns_base_chr_devt, 0,
+			NVME_MINORS, "nvmec");
+	if (result < 0)
+		goto unregister_dev_chrdev;
+
 	nvme_class = class_create(THIS_MODULE, "nvme");
 	if (IS_ERR(nvme_class)) {
 		result = PTR_ERR(nvme_class);
-		goto unregister_chrdev;
+		goto unregister_ns_chrdev;
 	}
 	nvme_class->dev_uevent = nvme_class_uevent;
 
+	nvme_ns_class = class_create(THIS_MODULE, "nvme-ns");
+	if (IS_ERR(nvme_ns_class)) {
+		result = PTR_ERR(nvme_ns_class);
+		goto destroy_dev_class;
+	}
+
 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
 	if (IS_ERR(nvme_subsys_class)) {
 		result = PTR_ERR(nvme_subsys_class);
-		goto destroy_class;
+		goto destroy_ns_class;
 	}
 	return 0;
 
-destroy_class:
+destroy_ns_class:
+	class_destroy(nvme_ns_class);
+destroy_dev_class:
 	class_destroy(nvme_class);
-unregister_chrdev:
+unregister_ns_chrdev:
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
+unregister_dev_chrdev:
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 destroy_delete_wq:
 	destroy_workqueue(nvme_delete_wq);
@@ -4765,6 +4874,7 @@ static void __exit nvme_core_exit(void)
 {
 	class_destroy(nvme_subsys_class);
 	class_destroy(nvme_class);
+	unregister_chrdev_region(nvme_ns_base_chr_devt, NVME_MINORS);
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
 	destroy_workqueue(nvme_delete_wq);
 	destroy_workqueue(nvme_reset_wq);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index bfcedfa4b057..1dd99f207aee 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -439,6 +439,9 @@ struct nvme_ns {
 	struct kref kref;
 	struct nvme_ns_head *head;
 
+	struct device cdev_device;	/* char device */
+	struct cdev cdev;
+
 	int lba_shift;
 	u16 ms;
 	u16 sgs;
@@ -818,6 +821,11 @@ static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
 	return dev_to_disk(dev)->private_data;
 }
 
+static inline struct nvme_ns *nvme_get_ns_from_cdev(struct device *dev)
+{
+	return dev_get_drvdata(dev);
+}
+
 #ifdef CONFIG_NVME_HWMON
 int nvme_hwmon_init(struct nvme_ctrl *ctrl);
 #else
-- 
2.17.1


_______________________________________________
Linux-nvme mailing list
Linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme