* [PATCH 1/9] nvme: move block_device_operations and ns/ctrl freeing to common code
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 2/9] nvme: add explicit quirk handling Christoph Hellwig
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
This moves the block_device_operations over to common code mostly
as-is. The only change is that the ns and ctrl refcounting got some
small refcounting to have wrappers around the kref_put operations.
A new free_ctrl operation is added to allow the PCI driver to free
it's ressources on the final drop.
Signed-off-by: Christoph Hellwig <hch at lst.de>
[Moved the integrity and pr changes due to merge conflict]
Signed-off-by: Keith Busch <keith.busch at intel.com>
---
drivers/nvme/host/core.c | 413 +++++++++++++++++++++++++++++++++++++++++++++++
drivers/nvme/host/nvme.h | 14 ++
drivers/nvme/host/pci.c | 412 ++--------------------------------------------
3 files changed, 439 insertions(+), 400 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index cc28150..63ec86a 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -15,12 +15,55 @@
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
#include <linux/errno.h>
+#include <linux/hdreg.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/types.h>
+#include <linux/pr.h>
+#include <linux/ptrace.h>
+#include <linux/nvme_ioctl.h>
+#include <linux/t10-pi.h>
+#include <scsi/sg.h>
+#include <asm/unaligned.h>
#include "nvme.h"
+DEFINE_SPINLOCK(dev_list_lock);
+
+static void nvme_free_ns(struct kref *kref)
+{
+ struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
+
+ if (ns->type == NVME_NS_LIGHTNVM)
+ nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
+
+ spin_lock(&dev_list_lock);
+ ns->disk->private_data = NULL;
+ spin_unlock(&dev_list_lock);
+
+ nvme_put_ctrl(ns->ctrl);
+ put_disk(ns->disk);
+ kfree(ns);
+}
+
+void nvme_put_ns(struct nvme_ns *ns)
+{
+ kref_put(&ns->kref, nvme_free_ns);
+}
+
+static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
+{
+ struct nvme_ns *ns;
+
+ spin_lock(&dev_list_lock);
+ ns = disk->private_data;
+ if (ns && !kref_get_unless_zero(&ns->kref))
+ ns = NULL;
+ spin_unlock(&dev_list_lock);
+
+ return ns;
+}
+
struct request *nvme_alloc_request(struct request_queue *q,
struct nvme_command *cmd, unsigned int flags)
{
@@ -269,3 +312,373 @@ int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
kfree(*log);
return error;
}
+
+static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
+{
+ struct nvme_user_io io;
+ struct nvme_command c;
+ unsigned length, meta_len;
+ void __user *metadata;
+
+ if (copy_from_user(&io, uio, sizeof(io)))
+ return -EFAULT;
+
+ switch (io.opcode) {
+ case nvme_cmd_write:
+ case nvme_cmd_read:
+ case nvme_cmd_compare:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ length = (io.nblocks + 1) << ns->lba_shift;
+ meta_len = (io.nblocks + 1) * ns->ms;
+ metadata = (void __user *)(uintptr_t)io.metadata;
+
+ if (ns->ext) {
+ length += meta_len;
+ meta_len = 0;
+ } else if (meta_len) {
+ if ((io.metadata & 3) || !io.metadata)
+ return -EINVAL;
+ }
+
+ memset(&c, 0, sizeof(c));
+ c.rw.opcode = io.opcode;
+ c.rw.flags = io.flags;
+ c.rw.nsid = cpu_to_le32(ns->ns_id);
+ c.rw.slba = cpu_to_le64(io.slba);
+ c.rw.length = cpu_to_le16(io.nblocks);
+ c.rw.control = cpu_to_le16(io.control);
+ c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
+ c.rw.reftag = cpu_to_le32(io.reftag);
+ c.rw.apptag = cpu_to_le16(io.apptag);
+ c.rw.appmask = cpu_to_le16(io.appmask);
+
+ return __nvme_submit_user_cmd(ns->queue, &c,
+ (void __user *)(uintptr_t)io.addr, length,
+ metadata, meta_len, io.slba, NULL, 0);
+}
+
+int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
+ struct nvme_passthru_cmd __user *ucmd)
+{
+ struct nvme_passthru_cmd cmd;
+ struct nvme_command c;
+ unsigned timeout = 0;
+ int status;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EACCES;
+ if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
+ return -EFAULT;
+
+ memset(&c, 0, sizeof(c));
+ c.common.opcode = cmd.opcode;
+ c.common.flags = cmd.flags;
+ c.common.nsid = cpu_to_le32(cmd.nsid);
+ c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
+ c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
+ c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
+ c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
+ c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
+ c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
+ c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
+ c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
+
+ if (cmd.timeout_ms)
+ timeout = msecs_to_jiffies(cmd.timeout_ms);
+
+ status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
+ (void __user *)cmd.addr, cmd.data_len,
+ &cmd.result, timeout);
+ if (status >= 0) {
+ if (put_user(cmd.result, &ucmd->result))
+ return -EFAULT;
+ }
+
+ return status;
+}
+
+static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
+ unsigned int cmd, unsigned long arg)
+{
+ struct nvme_ns *ns = bdev->bd_disk->private_data;
+
+ switch (cmd) {
+ case NVME_IOCTL_ID:
+ force_successful_syscall_return();
+ return ns->ns_id;
+ case NVME_IOCTL_ADMIN_CMD:
+ return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
+ case NVME_IOCTL_IO_CMD:
+ return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
+ case NVME_IOCTL_SUBMIT_IO:
+ return nvme_submit_io(ns, (void __user *)arg);
+ case SG_GET_VERSION_NUM:
+ return nvme_sg_get_version_num((void __user *)arg);
+ case SG_IO:
+ return nvme_sg_io(ns, (void __user *)arg);
+ default:
+ return -ENOTTY;
+ }
+}
+
+#ifdef CONFIG_COMPAT
+static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
+ unsigned int cmd, unsigned long arg)
+{
+ switch (cmd) {
+ case SG_IO:
+ return -ENOIOCTLCMD;
+ }
+ return nvme_ioctl(bdev, mode, cmd, arg);
+}
+#else
+#define nvme_compat_ioctl NULL
+#endif
+
+static int nvme_open(struct block_device *bdev, fmode_t mode)
+{
+ return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
+}
+
+static void nvme_release(struct gendisk *disk, fmode_t mode)
+{
+ nvme_put_ns(disk->private_data);
+}
+
+static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
+{
+ /* some standard values */
+ geo->heads = 1 << 6;
+ geo->sectors = 1 << 5;
+ geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
+ return 0;
+}
+
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+static void nvme_init_integrity(struct nvme_ns *ns)
+{
+ struct blk_integrity integrity;
+
+ switch (ns->pi_type) {
+ case NVME_NS_DPS_PI_TYPE3:
+ integrity.profile = &t10_pi_type3_crc;
+ break;
+ case NVME_NS_DPS_PI_TYPE1:
+ case NVME_NS_DPS_PI_TYPE2:
+ integrity.profile = &t10_pi_type1_crc;
+ break;
+ default:
+ integrity.profile = NULL;
+ break;
+ }
+ integrity.tuple_size = ns->ms;
+ blk_integrity_register(ns->disk, &integrity);
+ blk_queue_max_integrity_segments(ns->queue, 1);
+}
+#else
+static void nvme_init_integrity(struct nvme_ns *ns)
+{
+}
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
+static void nvme_config_discard(struct nvme_ns *ns)
+{
+ u32 logical_block_size = queue_logical_block_size(ns->queue);
+ ns->queue->limits.discard_zeroes_data = 0;
+ ns->queue->limits.discard_alignment = logical_block_size;
+ ns->queue->limits.discard_granularity = logical_block_size;
+ blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
+ queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
+}
+
+int nvme_revalidate_disk(struct gendisk *disk)
+{
+ struct nvme_ns *ns = disk->private_data;
+ struct nvme_id_ns *id;
+ u8 lbaf, pi_type;
+ u16 old_ms;
+ unsigned short bs;
+
+ if (nvme_identify_ns(ns->ctrl, ns->ns_id, &id)) {
+ dev_warn(ns->ctrl->dev, "%s: Identify failure nvme%dn%d\n",
+ __func__, ns->ctrl->instance, ns->ns_id);
+ return -ENODEV;
+ }
+ if (id->ncap == 0) {
+ kfree(id);
+ return -ENODEV;
+ }
+
+ if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
+ if (nvme_nvm_register(ns->queue, disk->disk_name)) {
+ dev_warn(ns->ctrl->dev,
+ "%s: LightNVM init failure\n", __func__);
+ kfree(id);
+ return -ENODEV;
+ }
+ ns->type = NVME_NS_LIGHTNVM;
+ }
+
+ old_ms = ns->ms;
+ lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
+ ns->lba_shift = id->lbaf[lbaf].ds;
+ ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
+ ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
+
+ /*
+ * If identify namespace failed, use default 512 byte block size so
+ * block layer can use before failing read/write for 0 capacity.
+ */
+ if (ns->lba_shift == 0)
+ ns->lba_shift = 9;
+ bs = 1 << ns->lba_shift;
+
+ /* XXX: PI implementation requires metadata equal t10 pi tuple size */
+ pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
+ id->dps & NVME_NS_DPS_PI_MASK : 0;
+
+ blk_mq_freeze_queue(disk->queue);
+ if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
+ ns->ms != old_ms ||
+ bs != queue_logical_block_size(disk->queue) ||
+ (ns->ms && ns->ext)))
+ blk_integrity_unregister(disk);
+
+ ns->pi_type = pi_type;
+ blk_queue_logical_block_size(ns->queue, bs);
+
+ if (ns->ms && !ns->ext)
+ nvme_init_integrity(ns);
+
+ if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
+ set_capacity(disk, 0);
+ else
+ set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
+
+ if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
+ nvme_config_discard(ns);
+ blk_mq_unfreeze_queue(disk->queue);
+
+ kfree(id);
+ return 0;
+}
+
+static char nvme_pr_type(enum pr_type type)
+{
+ switch (type) {
+ case PR_WRITE_EXCLUSIVE:
+ return 1;
+ case PR_EXCLUSIVE_ACCESS:
+ return 2;
+ case PR_WRITE_EXCLUSIVE_REG_ONLY:
+ return 3;
+ case PR_EXCLUSIVE_ACCESS_REG_ONLY:
+ return 4;
+ case PR_WRITE_EXCLUSIVE_ALL_REGS:
+ return 5;
+ case PR_EXCLUSIVE_ACCESS_ALL_REGS:
+ return 6;
+ default:
+ return 0;
+ }
+};
+
+static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
+ u64 key, u64 sa_key, u8 op)
+{
+ struct nvme_ns *ns = bdev->bd_disk->private_data;
+ struct nvme_command c;
+ u8 data[16] = { 0, };
+
+ put_unaligned_le64(key, &data[0]);
+ put_unaligned_le64(sa_key, &data[8]);
+
+ memset(&c, 0, sizeof(c));
+ c.common.opcode = op;
+ c.common.nsid = cpu_to_le32(ns->ns_id);
+ c.common.cdw10[0] = cpu_to_le32(cdw10);
+
+ return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
+}
+
+static int nvme_pr_register(struct block_device *bdev, u64 old,
+ u64 new, unsigned flags)
+{
+ u32 cdw10;
+
+ if (flags & ~PR_FL_IGNORE_KEY)
+ return -EOPNOTSUPP;
+
+ cdw10 = old ? 2 : 0;
+ cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
+ cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
+ return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
+}
+
+static int nvme_pr_reserve(struct block_device *bdev, u64 key,
+ enum pr_type type, unsigned flags)
+{
+ u32 cdw10;
+
+ if (flags & ~PR_FL_IGNORE_KEY)
+ return -EOPNOTSUPP;
+
+ cdw10 = nvme_pr_type(type) << 8;
+ cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
+}
+
+static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
+ enum pr_type type, bool abort)
+{
+ u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
+ return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
+}
+
+static int nvme_pr_clear(struct block_device *bdev, u64 key)
+{
+ u32 cdw10 = 1 | key ? 1 << 3 : 0;
+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
+}
+
+static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
+{
+ u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
+}
+
+static const struct pr_ops nvme_pr_ops = {
+ .pr_register = nvme_pr_register,
+ .pr_reserve = nvme_pr_reserve,
+ .pr_release = nvme_pr_release,
+ .pr_preempt = nvme_pr_preempt,
+ .pr_clear = nvme_pr_clear,
+};
+
+const struct block_device_operations nvme_fops = {
+ .owner = THIS_MODULE,
+ .ioctl = nvme_ioctl,
+ .compat_ioctl = nvme_compat_ioctl,
+ .open = nvme_open,
+ .release = nvme_release,
+ .getgeo = nvme_getgeo,
+ .revalidate_disk= nvme_revalidate_disk,
+ .pr_ops = &nvme_pr_ops,
+};
+
+static void nvme_free_ctrl(struct kref *kref)
+{
+ struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
+
+ ctrl->ops->free_ctrl(ctrl);
+}
+
+void nvme_put_ctrl(struct nvme_ctrl *ctrl)
+{
+ kref_put(&ctrl->kref, nvme_free_ctrl);
+}
+
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 5ba9acb..3b3f855 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -19,6 +19,8 @@
#include <linux/kref.h>
#include <linux/blk-mq.h>
+struct nvme_passthru_cmd;
+
extern unsigned char nvme_io_timeout;
#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
@@ -34,6 +36,7 @@ struct nvme_ctrl {
const struct nvme_ctrl_ops *ops;
struct request_queue *admin_q;
struct device *dev;
+ struct kref kref;
int instance;
char name[12];
@@ -69,6 +72,7 @@ struct nvme_ns {
struct nvme_ctrl_ops {
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
+ void (*free_ctrl)(struct nvme_ctrl *ctrl);
};
static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
@@ -148,6 +152,9 @@ static inline int nvme_error_status(u16 status)
}
}
+void nvme_put_ctrl(struct nvme_ctrl *ctrl);
+void nvme_put_ns(struct nvme_ns *ns);
+
struct request *nvme_alloc_request(struct request_queue *q,
struct nvme_command *cmd, unsigned int flags);
int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
@@ -170,6 +177,13 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
dma_addr_t dma_addr, u32 *result);
+extern const struct block_device_operations nvme_fops;
+extern spinlock_t dev_list_lock;
+
+int nvme_revalidate_disk(struct gendisk *disk);
+int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
+ struct nvme_passthru_cmd __user *ucmd);
+
struct sg_io_hdr;
int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index aa033f0..e0f40af 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -79,7 +79,6 @@ static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
-static DEFINE_SPINLOCK(dev_list_lock);
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
static struct workqueue_struct *nvme_workq;
@@ -127,7 +126,6 @@ struct nvme_dev {
struct msix_entry *entry;
void __iomem *bar;
struct list_head namespaces;
- struct kref kref;
struct device *device;
struct work_struct reset_work;
struct work_struct probe_work;
@@ -601,27 +599,6 @@ static void nvme_dif_remap(struct request *req,
}
kunmap_atomic(pmap);
}
-
-static void nvme_init_integrity(struct nvme_ns *ns)
-{
- struct blk_integrity integrity;
-
- switch (ns->pi_type) {
- case NVME_NS_DPS_PI_TYPE3:
- integrity.profile = &t10_pi_type3_crc;
- break;
- case NVME_NS_DPS_PI_TYPE1:
- case NVME_NS_DPS_PI_TYPE2:
- integrity.profile = &t10_pi_type1_crc;
- break;
- default:
- integrity.profile = NULL;
- break;
- }
- integrity.tuple_size = ns->ms;
- blk_integrity_register(ns->disk, &integrity);
- blk_queue_max_integrity_segments(ns->queue, 1);
-}
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
@@ -633,9 +610,6 @@ static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
-static void nvme_init_integrity(struct nvme_ns *ns)
-{
-}
#endif
static void req_completion(struct nvme_queue *nvmeq, void *ctx,
@@ -1633,94 +1607,6 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
return result;
}
-static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
-{
- struct nvme_user_io io;
- struct nvme_command c;
- unsigned length, meta_len;
- void __user *metadata;
-
- if (copy_from_user(&io, uio, sizeof(io)))
- return -EFAULT;
-
- switch (io.opcode) {
- case nvme_cmd_write:
- case nvme_cmd_read:
- case nvme_cmd_compare:
- break;
- default:
- return -EINVAL;
- }
-
- length = (io.nblocks + 1) << ns->lba_shift;
- meta_len = (io.nblocks + 1) * ns->ms;
- metadata = (void __user *)(uintptr_t)io.metadata;
-
- if (ns->ext) {
- length += meta_len;
- meta_len = 0;
- } else if (meta_len) {
- if ((io.metadata & 3) || !io.metadata)
- return -EINVAL;
- }
-
- memset(&c, 0, sizeof(c));
- c.rw.opcode = io.opcode;
- c.rw.flags = io.flags;
- c.rw.nsid = cpu_to_le32(ns->ns_id);
- c.rw.slba = cpu_to_le64(io.slba);
- c.rw.length = cpu_to_le16(io.nblocks);
- c.rw.control = cpu_to_le16(io.control);
- c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
- c.rw.reftag = cpu_to_le32(io.reftag);
- c.rw.apptag = cpu_to_le16(io.apptag);
- c.rw.appmask = cpu_to_le16(io.appmask);
-
- return __nvme_submit_user_cmd(ns->queue, &c,
- (void __user *)(uintptr_t)io.addr, length,
- metadata, meta_len, io.slba, NULL, 0);
-}
-
-static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
- struct nvme_passthru_cmd __user *ucmd)
-{
- struct nvme_passthru_cmd cmd;
- struct nvme_command c;
- unsigned timeout = 0;
- int status;
-
- if (!capable(CAP_SYS_ADMIN))
- return -EACCES;
- if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
- return -EFAULT;
-
- memset(&c, 0, sizeof(c));
- c.common.opcode = cmd.opcode;
- c.common.flags = cmd.flags;
- c.common.nsid = cpu_to_le32(cmd.nsid);
- c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
- c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
- c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
- c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
- c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
- c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
- c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
- c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
-
- if (cmd.timeout_ms)
- timeout = msecs_to_jiffies(cmd.timeout_ms);
-
- status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
- (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
- &cmd.result, timeout);
- if (status >= 0) {
- if (put_user(cmd.result, &ucmd->result))
- return -EFAULT;
- }
-
- return status;
-}
-
static int nvme_subsys_reset(struct nvme_dev *dev)
{
if (!dev->subsystem)
@@ -1730,281 +1616,6 @@ static int nvme_subsys_reset(struct nvme_dev *dev)
return 0;
}
-static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
- unsigned long arg)
-{
- struct nvme_ns *ns = bdev->bd_disk->private_data;
-
- switch (cmd) {
- case NVME_IOCTL_ID:
- force_successful_syscall_return();
- return ns->ns_id;
- case NVME_IOCTL_ADMIN_CMD:
- return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
- case NVME_IOCTL_IO_CMD:
- return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
- case NVME_IOCTL_SUBMIT_IO:
- return nvme_submit_io(ns, (void __user *)arg);
- case SG_GET_VERSION_NUM:
- return nvme_sg_get_version_num((void __user *)arg);
- case SG_IO:
- return nvme_sg_io(ns, (void __user *)arg);
- default:
- return -ENOTTY;
- }
-}
-
-#ifdef CONFIG_COMPAT
-static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
- unsigned int cmd, unsigned long arg)
-{
- switch (cmd) {
- case SG_IO:
- return -ENOIOCTLCMD;
- }
- return nvme_ioctl(bdev, mode, cmd, arg);
-}
-#else
-#define nvme_compat_ioctl NULL
-#endif
-
-static void nvme_free_dev(struct kref *kref);
-static void nvme_free_ns(struct kref *kref)
-{
- struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
- struct nvme_dev *dev = to_nvme_dev(ns->ctrl);
-
- if (ns->type == NVME_NS_LIGHTNVM)
- nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
-
- spin_lock(&dev_list_lock);
- ns->disk->private_data = NULL;
- spin_unlock(&dev_list_lock);
-
- kref_put(&dev->kref, nvme_free_dev);
- put_disk(ns->disk);
- kfree(ns);
-}
-
-static int nvme_open(struct block_device *bdev, fmode_t mode)
-{
- int ret = 0;
- struct nvme_ns *ns;
-
- spin_lock(&dev_list_lock);
- ns = bdev->bd_disk->private_data;
- if (!ns)
- ret = -ENXIO;
- else if (!kref_get_unless_zero(&ns->kref))
- ret = -ENXIO;
- spin_unlock(&dev_list_lock);
-
- return ret;
-}
-
-static void nvme_release(struct gendisk *disk, fmode_t mode)
-{
- struct nvme_ns *ns = disk->private_data;
- kref_put(&ns->kref, nvme_free_ns);
-}
-
-static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
-{
- /* some standard values */
- geo->heads = 1 << 6;
- geo->sectors = 1 << 5;
- geo->cylinders = get_capacity(bd->bd_disk) >> 11;
- return 0;
-}
-
-static void nvme_config_discard(struct nvme_ns *ns)
-{
- u32 logical_block_size = queue_logical_block_size(ns->queue);
- ns->queue->limits.discard_zeroes_data = 0;
- ns->queue->limits.discard_alignment = logical_block_size;
- ns->queue->limits.discard_granularity = logical_block_size;
- blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
- queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
-}
-
-static int nvme_revalidate_disk(struct gendisk *disk)
-{
- struct nvme_ns *ns = disk->private_data;
- struct nvme_dev *dev = to_nvme_dev(ns->ctrl);
- struct nvme_id_ns *id;
- u8 lbaf, pi_type;
- u16 old_ms;
- unsigned short bs;
-
- if (nvme_identify_ns(&dev->ctrl, ns->ns_id, &id)) {
- dev_warn(dev->dev, "%s: Identify failure nvme%dn%d\n", __func__,
- dev->ctrl.instance, ns->ns_id);
- return -ENODEV;
- }
- if (id->ncap == 0) {
- kfree(id);
- return -ENODEV;
- }
-
- if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
- if (nvme_nvm_register(ns->queue, disk->disk_name)) {
- dev_warn(dev->dev,
- "%s: LightNVM init failure\n", __func__);
- kfree(id);
- return -ENODEV;
- }
- ns->type = NVME_NS_LIGHTNVM;
- }
-
- old_ms = ns->ms;
- lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
- ns->lba_shift = id->lbaf[lbaf].ds;
- ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
- ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
-
- /*
- * If identify namespace failed, use default 512 byte block size so
- * block layer can use before failing read/write for 0 capacity.
- */
- if (ns->lba_shift == 0)
- ns->lba_shift = 9;
- bs = 1 << ns->lba_shift;
-
- /* XXX: PI implementation requires metadata equal t10 pi tuple size */
- pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
- id->dps & NVME_NS_DPS_PI_MASK : 0;
-
- blk_mq_freeze_queue(disk->queue);
- if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
- ns->ms != old_ms ||
- bs != queue_logical_block_size(disk->queue) ||
- (ns->ms && ns->ext)))
- blk_integrity_unregister(disk);
-
- ns->pi_type = pi_type;
- blk_queue_logical_block_size(ns->queue, bs);
-
- if (ns->ms && !ns->ext)
- nvme_init_integrity(ns);
-
- if ((ns->ms && !(ns->ms == 8 && ns->pi_type) &&
- !blk_get_integrity(disk)) ||
- ns->type == NVME_NS_LIGHTNVM)
- set_capacity(disk, 0);
- else
- set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
-
- if (dev->ctrl.oncs & NVME_CTRL_ONCS_DSM)
- nvme_config_discard(ns);
- blk_mq_unfreeze_queue(disk->queue);
-
- kfree(id);
- return 0;
-}
-
-static char nvme_pr_type(enum pr_type type)
-{
- switch (type) {
- case PR_WRITE_EXCLUSIVE:
- return 1;
- case PR_EXCLUSIVE_ACCESS:
- return 2;
- case PR_WRITE_EXCLUSIVE_REG_ONLY:
- return 3;
- case PR_EXCLUSIVE_ACCESS_REG_ONLY:
- return 4;
- case PR_WRITE_EXCLUSIVE_ALL_REGS:
- return 5;
- case PR_EXCLUSIVE_ACCESS_ALL_REGS:
- return 6;
- default:
- return 0;
- }
-};
-
-static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
- u64 key, u64 sa_key, u8 op)
-{
- struct nvme_ns *ns = bdev->bd_disk->private_data;
- struct nvme_command c;
- u8 data[16] = { 0, };
-
- put_unaligned_le64(key, &data[0]);
- put_unaligned_le64(sa_key, &data[8]);
-
- memset(&c, 0, sizeof(c));
- c.common.opcode = op;
- c.common.nsid = cpu_to_le32(ns->ns_id);
- c.common.cdw10[0] = cpu_to_le32(cdw10);
-
- return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
-}
-
-static int nvme_pr_register(struct block_device *bdev, u64 old,
- u64 new, unsigned flags)
-{
- u32 cdw10;
-
- if (flags & ~PR_FL_IGNORE_KEY)
- return -EOPNOTSUPP;
-
- cdw10 = old ? 2 : 0;
- cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
- cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
- return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
-}
-
-static int nvme_pr_reserve(struct block_device *bdev, u64 key,
- enum pr_type type, unsigned flags)
-{
- u32 cdw10;
-
- if (flags & ~PR_FL_IGNORE_KEY)
- return -EOPNOTSUPP;
-
- cdw10 = nvme_pr_type(type) << 8;
- cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
- return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
-}
-
-static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
- enum pr_type type, bool abort)
-{
- u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
- return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
-}
-
-static int nvme_pr_clear(struct block_device *bdev, u64 key)
-{
- u32 cdw10 = 1 | (key ? 1 << 3 : 0);
- return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
-}
-
-static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
-{
- u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
- return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
-}
-
-static const struct pr_ops nvme_pr_ops = {
- .pr_register = nvme_pr_register,
- .pr_reserve = nvme_pr_reserve,
- .pr_release = nvme_pr_release,
- .pr_preempt = nvme_pr_preempt,
- .pr_clear = nvme_pr_clear,
-};
-
-static const struct block_device_operations nvme_fops = {
- .owner = THIS_MODULE,
- .ioctl = nvme_ioctl,
- .compat_ioctl = nvme_compat_ioctl,
- .open = nvme_open,
- .release = nvme_release,
- .getgeo = nvme_getgeo,
- .revalidate_disk= nvme_revalidate_disk,
- .pr_ops = &nvme_pr_ops,
-};
-
static int nvme_kthread(void *data)
{
struct nvme_dev *dev, *next;
@@ -2105,7 +1716,7 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
if (nvme_revalidate_disk(ns->disk))
goto out_free_disk;
- kref_get(&dev->kref);
+ kref_get(&dev->ctrl.kref);
if (ns->type != NVME_NS_LIGHTNVM) {
add_disk(ns->disk);
if (ns->ms) {
@@ -2354,7 +1965,7 @@ static void nvme_ns_remove(struct nvme_ns *ns)
blk_cleanup_queue(ns->queue);
}
list_del_init(&ns->list);
- kref_put(&ns->kref, nvme_free_ns);
+ nvme_put_ns(ns);
}
static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
@@ -2828,9 +2439,9 @@ static void nvme_release_instance(struct nvme_dev *dev)
spin_unlock(&dev_list_lock);
}
-static void nvme_free_dev(struct kref *kref)
+static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
{
- struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
+ struct nvme_dev *dev = to_nvme_dev(ctrl);
put_device(dev->dev);
put_device(dev->device);
@@ -2857,7 +2468,7 @@ static int nvme_dev_open(struct inode *inode, struct file *f)
ret = -EWOULDBLOCK;
break;
}
- if (!kref_get_unless_zero(&dev->kref))
+ if (!kref_get_unless_zero(&dev->ctrl.kref))
break;
f->private_data = dev;
ret = 0;
@@ -2872,7 +2483,7 @@ static int nvme_dev_open(struct inode *inode, struct file *f)
static int nvme_dev_release(struct inode *inode, struct file *f)
{
struct nvme_dev *dev = f->private_data;
- kref_put(&dev->kref, nvme_free_dev);
+ nvme_put_ctrl(&dev->ctrl);
return 0;
}
@@ -2987,19 +2598,19 @@ static int nvme_remove_dead_ctrl(void *arg)
if (pci_get_drvdata(pdev))
pci_stop_and_remove_bus_device_locked(pdev);
- kref_put(&dev->kref, nvme_free_dev);
+ nvme_put_ctrl(&dev->ctrl);
return 0;
}
static void nvme_dead_ctrl(struct nvme_dev *dev)
{
dev_warn(dev->dev, "Device failed to resume\n");
- kref_get(&dev->kref);
+ kref_get(&dev->ctrl.kref);
if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
dev->ctrl.instance))) {
dev_err(dev->dev,
"Failed to start controller remove task\n");
- kref_put(&dev->kref, nvme_free_dev);
+ nvme_put_ctrl(&dev->ctrl);
}
}
@@ -3077,6 +2688,7 @@ static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.reg_read32 = nvme_pci_reg_read32,
+ .free_ctrl = nvme_pci_free_ctrl,
};
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -3116,7 +2728,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (result)
goto release;
- kref_init(&dev->kref);
+ kref_init(&dev->ctrl.kref);
dev->device = device_create(nvme_class, &pdev->dev,
MKDEV(nvme_char_major, dev->ctrl.instance),
dev, "nvme%d", dev->ctrl.instance);
@@ -3189,7 +2801,7 @@ static void nvme_remove(struct pci_dev *pdev)
nvme_free_queues(dev, 0);
nvme_release_cmb(dev);
nvme_release_prp_pools(dev);
- kref_put(&dev->kref, nvme_free_dev);
+ nvme_put_ctrl(&dev->ctrl);
}
/* These functions are yet to be implemented */
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 2/9] nvme: add explicit quirk handling
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
2015-11-30 8:40 ` [PATCH 1/9] nvme: move block_device_operations and ns/ctrl freeing to common code Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 3/9] nvme: move remaining CC setup into nvme_enable_ctrl Christoph Hellwig
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
Add an enum for all workarounds not in the spec and identify the affected
controllers at probe time.
Signed-off-by: Christoph Hellwig <hch at lst.de>
Signed-off-by: Keith Busch <keith.busch at intel.com>
---
drivers/nvme/host/nvme.h | 13 +++++++++++++
drivers/nvme/host/pci.c | 8 +++++---
2 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 3b3f855..f7f16e3 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -32,6 +32,18 @@ enum {
NVME_NS_LIGHTNVM = 1,
};
+/*
+ * List of workarounds for devices that required behavior not specified in
+ * the standard.
+ */
+enum nvme_quirks {
+ /*
+ * Prefers I/O aligned to a stripe size specified in a vendor
+ * specific Identify field.
+ */
+ NVME_QUIRK_STRIPE_SIZE = (1 << 0),
+};
+
struct nvme_ctrl {
const struct nvme_ctrl_ops *ops;
struct request_queue *admin_q;
@@ -47,6 +59,7 @@ struct nvme_ctrl {
u16 abort_limit;
u8 event_limit;
u8 vwc;
+ unsigned long quirks;
};
/*
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index e0f40af..27d7449 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -2026,7 +2026,6 @@ static void nvme_dev_scan(struct work_struct *work)
*/
static int nvme_dev_add(struct nvme_dev *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev->dev);
int res;
struct nvme_id_ctrl *ctrl;
int shift = NVME_CAP_MPSMIN(lo_hi_readq(dev->bar + NVME_REG_CAP)) + 12;
@@ -2047,8 +2046,8 @@ static int nvme_dev_add(struct nvme_dev *dev)
dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
else
dev->max_hw_sectors = UINT_MAX;
- if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
- (pdev->device == 0x0953) && ctrl->vs[3]) {
+
+ if ((dev->ctrl.quirks & NVME_QUIRK_STRIPE_SIZE) && ctrl->vs[3]) {
unsigned int max_hw_sectors;
dev->stripe_size = 1 << (ctrl->vs[3] + shift);
@@ -2719,6 +2718,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
dev->ctrl.ops = &nvme_pci_ctrl_ops;
dev->ctrl.dev = dev->dev;
+ dev->ctrl.quirks = id->driver_data;
result = nvme_set_instance(dev);
if (result)
@@ -2846,6 +2846,8 @@ static const struct pci_error_handlers nvme_err_handler = {
#define PCI_CLASS_STORAGE_EXPRESS 0x010802
static const struct pci_device_id nvme_id_table[] = {
+ { PCI_VDEVICE(INTEL, 0x0953),
+ .driver_data = NVME_QUIRK_STRIPE_SIZE, },
{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
{ 0, }
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 3/9] nvme: move remaining CC setup into nvme_enable_ctrl
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
2015-11-30 8:40 ` [PATCH 1/9] nvme: move block_device_operations and ns/ctrl freeing to common code Christoph Hellwig
2015-11-30 8:40 ` [PATCH 2/9] nvme: add explicit quirk handling Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 4/9] nvme: move nvme_{enable, disable, shutdown}_ctrl to common code Christoph Hellwig
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
Remove the calculation of all the bits written into the CC register into
nvme_enable_ctrl, so that they can be moved into the core NVMe driver in
the future.
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
drivers/nvme/host/pci.c | 44 +++++++++++++++++++++-----------------------
1 file changed, 21 insertions(+), 23 deletions(-)
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 27d7449..1ae94cd 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1446,8 +1446,28 @@ static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
{
- dev->ctrl_config &= ~NVME_CC_SHN_MASK;
+ /*
+ * Default to a 4K page size, with the intention to update this
+ * path in the future to accomodate architectures with differing
+ * kernel and IO page sizes.
+ */
+ unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
+
+ if (page_shift < dev_page_min) {
+ dev_err(dev->dev,
+ "Minimum device page size %u too large for host (%u)\n",
+ 1 << dev_page_min, 1 << page_shift);
+ return -ENODEV;
+ }
+
+ dev->page_size = 1 << page_shift;
+
+ dev->ctrl_config = NVME_CC_CSS_NVM;
+ dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
+ dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
+ dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
dev->ctrl_config |= NVME_CC_ENABLE;
+
writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
return nvme_wait_ready(dev, cap, true);
@@ -1541,21 +1561,6 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
u32 aqa;
u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
struct nvme_queue *nvmeq;
- /*
- * default to a 4K page size, with the intention to update this
- * path in the future to accomodate architectures with differing
- * kernel and IO page sizes.
- */
- unsigned page_shift = 12;
- unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
-
- if (page_shift < dev_page_min) {
- dev_err(dev->dev,
- "Minimum device page size (%u) too large for "
- "host (%u)\n", 1 << dev_page_min,
- 1 << page_shift);
- return -ENODEV;
- }
dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
NVME_CAP_NSSRC(cap) : 0;
@@ -1578,13 +1583,6 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
aqa = nvmeq->q_depth - 1;
aqa |= aqa << 16;
- dev->page_size = 1 << page_shift;
-
- dev->ctrl_config = NVME_CC_CSS_NVM;
- dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
- dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
- dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
-
writel(aqa, dev->bar + NVME_REG_AQA);
lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 4/9] nvme: move nvme_{enable, disable, shutdown}_ctrl to common code
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
` (2 preceding siblings ...)
2015-11-30 8:40 ` [PATCH 3/9] nvme: move remaining CC setup into nvme_enable_ctrl Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 5/9] nvme: add a common helper to read Identify Controller data Christoph Hellwig
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
drivers/nvme/host/core.c | 106 +++++++++++++++++++++++++++++++++++++
drivers/nvme/host/nvme.h | 11 ++++
drivers/nvme/host/pci.c | 133 +++++++++--------------------------------------
3 files changed, 141 insertions(+), 109 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 63ec86a..e3179b3 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -14,6 +14,7 @@
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
+#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/hdreg.h>
#include <linux/kernel.h>
@@ -670,6 +671,111 @@ const struct block_device_operations nvme_fops = {
.pr_ops = &nvme_pr_ops,
};
+static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
+{
+ unsigned long timeout =
+ ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
+ u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
+ int ret;
+
+ while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
+ if ((csts & NVME_CSTS_RDY) == bit)
+ break;
+
+ msleep(100);
+ if (fatal_signal_pending(current))
+ return -EINTR;
+ if (time_after(jiffies, timeout)) {
+ dev_err(ctrl->dev,
+ "Device not ready; aborting %s\n", enabled ?
+ "initialisation" : "reset");
+ return -ENODEV;
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * If the device has been passed off to us in an enabled state, just clear
+ * the enabled bit. The spec says we should set the 'shutdown notification
+ * bits', but doing so may cause the device to complete commands to the
+ * admin queue ... and we don't know what memory that might be pointing at!
+ */
+int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
+{
+ int ret;
+
+ ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
+ ctrl->ctrl_config &= ~NVME_CC_ENABLE;
+
+ ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
+ if (ret)
+ return ret;
+ return nvme_wait_ready(ctrl, cap, false);
+}
+
+int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
+{
+ /*
+ * Default to a 4K page size, with the intention to update this
+ * path in the future to accomodate architectures with differing
+ * kernel and IO page sizes.
+ */
+ unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
+ int ret;
+
+ if (page_shift < dev_page_min) {
+ dev_err(ctrl->dev,
+ "Minimum device page size %u too large for host (%u)\n",
+ 1 << dev_page_min, 1 << page_shift);
+ return -ENODEV;
+ }
+
+ ctrl->page_size = 1 << page_shift;
+
+ ctrl->ctrl_config = NVME_CC_CSS_NVM;
+ ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
+ ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
+ ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
+ ctrl->ctrl_config |= NVME_CC_ENABLE;
+
+ ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
+ if (ret)
+ return ret;
+ return nvme_wait_ready(ctrl, cap, true);
+}
+
+int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
+{
+ unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
+ u32 csts;
+ int ret;
+
+ ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
+ ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
+
+ ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
+ if (ret)
+ return ret;
+
+ while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
+ if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
+ break;
+
+ msleep(100);
+ if (fatal_signal_pending(current))
+ return -EINTR;
+ if (time_after(jiffies, timeout)) {
+ dev_err(ctrl->dev,
+ "Device shutdown incomplete; abort shutdown\n");
+ return -ENODEV;
+ }
+ }
+
+ return ret;
+}
+
static void nvme_free_ctrl(struct kref *kref)
{
struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index f7f16e3..b6c5a55 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -27,6 +27,9 @@ extern unsigned char nvme_io_timeout;
extern unsigned char admin_timeout;
#define ADMIN_TIMEOUT (admin_timeout * HZ)
+extern unsigned char shutdown_timeout;
+#define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ)
+
enum {
NVME_NS_LBA = 0,
NVME_NS_LIGHTNVM = 1,
@@ -55,6 +58,10 @@ struct nvme_ctrl {
char serial[20];
char model[40];
char firmware_rev[8];
+
+ u32 ctrl_config;
+
+ u32 page_size;
u16 oncs;
u16 abort_limit;
u8 event_limit;
@@ -85,6 +92,7 @@ struct nvme_ns {
struct nvme_ctrl_ops {
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
+ int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
void (*free_ctrl)(struct nvme_ctrl *ctrl);
};
@@ -165,6 +173,9 @@ static inline int nvme_error_status(u16 status)
}
}
+int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
+int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
+int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
void nvme_put_ctrl(struct nvme_ctrl *ctrl);
void nvme_put_ns(struct nvme_ns *ns);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 1ae94cd..ccb3151 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -52,7 +52,6 @@
#define NVME_AQ_DEPTH 256
#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
-#define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ)
unsigned char admin_timeout = 60;
module_param(admin_timeout, byte, 0644);
@@ -62,7 +61,7 @@ unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
-static unsigned char shutdown_timeout = 5;
+unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
@@ -122,7 +121,6 @@ struct nvme_dev {
unsigned max_qid;
int q_depth;
u32 db_stride;
- u32 ctrl_config;
struct msix_entry *entry;
void __iomem *bar;
struct list_head namespaces;
@@ -133,7 +131,6 @@ struct nvme_dev {
bool subsystem;
u32 max_hw_sectors;
u32 stripe_size;
- u32 page_size;
void __iomem *cmb;
dma_addr_t cmb_dma_addr;
u64 cmb_size;
@@ -225,7 +222,7 @@ struct nvme_cmd_info {
* Max size of iod being embedded in the request payload
*/
#define NVME_INT_PAGES 2
-#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->page_size)
+#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->ctrl.page_size)
#define NVME_INT_MASK 0x01
/*
@@ -235,7 +232,8 @@ struct nvme_cmd_info {
*/
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
- unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
+ unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
+ dev->ctrl.page_size);
return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}
@@ -527,7 +525,7 @@ static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
{
- const int last_prp = dev->page_size / 8 - 1;
+ const int last_prp = dev->ctrl.page_size / 8 - 1;
int i;
__le64 **list = iod_list(iod);
dma_addr_t prp_dma = iod->first_dma;
@@ -668,7 +666,7 @@ static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
struct scatterlist *sg = iod->sg;
int dma_len = sg_dma_len(sg);
u64 dma_addr = sg_dma_address(sg);
- u32 page_size = dev->page_size;
+ u32 page_size = dev->ctrl.page_size;
int offset = dma_addr & (page_size - 1);
__le64 *prp_list;
__le64 **list = iod_list(iod);
@@ -1275,11 +1273,12 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
int entry_size)
{
int q_depth = dev->q_depth;
- unsigned q_size_aligned = roundup(q_depth * entry_size, dev->page_size);
+ unsigned q_size_aligned = roundup(q_depth * entry_size,
+ dev->ctrl.page_size);
if (q_size_aligned * nr_io_queues > dev->cmb_size) {
u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
- mem_per_q = round_down(mem_per_q, dev->page_size);
+ mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
q_depth = div_u64(mem_per_q, entry_size);
/*
@@ -1298,8 +1297,8 @@ static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
int qid, int depth)
{
if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
- unsigned offset = (qid - 1) *
- roundup(SQ_SIZE(depth), dev->page_size);
+ unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
+ dev->ctrl.page_size);
nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
nvmeq->sq_cmds_io = dev->cmb + offset;
} else {
@@ -1407,97 +1406,6 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
return result;
}
-static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
-{
- unsigned long timeout;
- u32 bit = enabled ? NVME_CSTS_RDY : 0;
-
- timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
-
- while ((readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_RDY) != bit) {
- msleep(100);
- if (fatal_signal_pending(current))
- return -EINTR;
- if (time_after(jiffies, timeout)) {
- dev_err(dev->dev,
- "Device not ready; aborting %s\n", enabled ?
- "initialisation" : "reset");
- return -ENODEV;
- }
- }
-
- return 0;
-}
-
-/*
- * If the device has been passed off to us in an enabled state, just clear
- * the enabled bit. The spec says we should set the 'shutdown notification
- * bits', but doing so may cause the device to complete commands to the
- * admin queue ... and we don't know what memory that might be pointing at!
- */
-static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
-{
- dev->ctrl_config &= ~NVME_CC_SHN_MASK;
- dev->ctrl_config &= ~NVME_CC_ENABLE;
- writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
-
- return nvme_wait_ready(dev, cap, false);
-}
-
-static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
-{
- /*
- * Default to a 4K page size, with the intention to update this
- * path in the future to accomodate architectures with differing
- * kernel and IO page sizes.
- */
- unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
-
- if (page_shift < dev_page_min) {
- dev_err(dev->dev,
- "Minimum device page size %u too large for host (%u)\n",
- 1 << dev_page_min, 1 << page_shift);
- return -ENODEV;
- }
-
- dev->page_size = 1 << page_shift;
-
- dev->ctrl_config = NVME_CC_CSS_NVM;
- dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
- dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
- dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
- dev->ctrl_config |= NVME_CC_ENABLE;
-
- writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
-
- return nvme_wait_ready(dev, cap, true);
-}
-
-static int nvme_shutdown_ctrl(struct nvme_dev *dev)
-{
- unsigned long timeout;
-
- dev->ctrl_config &= ~NVME_CC_SHN_MASK;
- dev->ctrl_config |= NVME_CC_SHN_NORMAL;
-
- writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
-
- timeout = SHUTDOWN_TIMEOUT + jiffies;
- while ((readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_SHST_MASK) !=
- NVME_CSTS_SHST_CMPLT) {
- msleep(100);
- if (fatal_signal_pending(current))
- return -EINTR;
- if (time_after(jiffies, timeout)) {
- dev_err(dev->dev,
- "Device shutdown incomplete; abort shutdown\n");
- return -ENODEV;
- }
- }
-
- return 0;
-}
-
static struct blk_mq_ops nvme_mq_admin_ops = {
.queue_rq = nvme_queue_rq,
.map_queue = blk_mq_map_queue,
@@ -1569,7 +1477,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
(readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
- result = nvme_disable_ctrl(dev, cap);
+ result = nvme_disable_ctrl(&dev->ctrl, cap);
if (result < 0)
return result;
@@ -1587,7 +1495,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
- result = nvme_enable_ctrl(dev, cap);
+ result = nvme_enable_ctrl(&dev->ctrl, cap);
if (result)
goto free_nvmeq;
@@ -1687,13 +1595,13 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
if (dev->max_hw_sectors) {
blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
blk_queue_max_segments(ns->queue,
- (dev->max_hw_sectors / (dev->page_size >> 9)) + 1);
+ (dev->max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1);
}
if (dev->stripe_size)
blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT)
blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
- blk_queue_virt_boundary(ns->queue, dev->page_size - 1);
+ blk_queue_virt_boundary(ns->queue, dev->ctrl.page_size - 1);
disk->major = nvme_major;
disk->first_minor = 0;
@@ -2181,7 +2089,7 @@ static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
* queues than admin tags.
*/
set_current_state(TASK_RUNNING);
- nvme_disable_ctrl(dev,
+ nvme_disable_ctrl(&dev->ctrl,
lo_hi_readq(dev->bar + NVME_REG_CAP));
nvme_clear_queue(dev->queues[0]);
flush_kthread_worker(dq->worker);
@@ -2367,7 +2275,7 @@ static void nvme_dev_shutdown(struct nvme_dev *dev)
}
} else {
nvme_disable_io_queues(dev);
- nvme_shutdown_ctrl(dev);
+ nvme_shutdown_ctrl(&dev->ctrl);
nvme_disable_queue(dev, 0);
}
nvme_dev_unmap(dev);
@@ -2683,8 +2591,15 @@ static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
return 0;
}
+static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
+{
+ writel(val, to_nvme_dev(ctrl)->bar + off);
+ return 0;
+}
+
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.reg_read32 = nvme_pci_reg_read32,
+ .reg_write32 = nvme_pci_reg_write32,
.free_ctrl = nvme_pci_free_ctrl,
};
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 5/9] nvme: add a common helper to read Identify Controller data
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
` (3 preceding siblings ...)
2015-11-30 8:40 ` [PATCH 4/9] nvme: move nvme_{enable, disable, shutdown}_ctrl to common code Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 6/9] nvme: move the call to nvme_init_identify earlier Christoph Hellwig
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
And add the 64-bit register read operation for it.
Signed-off-by: Christoph Hellwig <hch at lst.de>
Signed-off-by: Keith Busch <keith.busch at intel.com>
---
drivers/nvme/host/core.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++
drivers/nvme/host/nvme.h | 4 ++++
drivers/nvme/host/pci.c | 53 ++++++++++++++----------------------------------
3 files changed, 71 insertions(+), 38 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index e3179b3..1c9f09c 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -776,6 +776,58 @@ int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
return ret;
}
+/*
+ * Initialize the cached copies of the Identify data and various controller
+ * register in our nvme_ctrl structure. This should be called as soon as
+ * the admin queue is fully up and running.
+ */
+int nvme_init_identify(struct nvme_ctrl *ctrl)
+{
+ struct nvme_id_ctrl *id;
+ u64 cap;
+ int ret, page_shift;
+
+ ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
+ if (ret) {
+ dev_err(ctrl->dev, "Reading CAP failed (%d)\n", ret);
+ return ret;
+ }
+ page_shift = NVME_CAP_MPSMIN(cap) + 12;
+
+ ret = nvme_identify_ctrl(ctrl, &id);
+ if (ret) {
+ dev_err(ctrl->dev, "Identify Controller failed (%d)\n", ret);
+ return -EIO;
+ }
+
+ ctrl->oncs = le16_to_cpup(&id->oncs);
+ ctrl->abort_limit = id->acl + 1;
+ ctrl->vwc = id->vwc;
+ memcpy(ctrl->serial, id->sn, sizeof(id->sn));
+ memcpy(ctrl->model, id->mn, sizeof(id->mn));
+ memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
+ if (id->mdts)
+ ctrl->max_hw_sectors = 1 << (id->mdts + page_shift - 9);
+ else
+ ctrl->max_hw_sectors = UINT_MAX;
+
+ if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && id->vs[3]) {
+ unsigned int max_hw_sectors;
+
+ ctrl->stripe_size = 1 << (id->vs[3] + page_shift);
+ max_hw_sectors = ctrl->stripe_size >> (page_shift - 9);
+ if (ctrl->max_hw_sectors) {
+ ctrl->max_hw_sectors = min(max_hw_sectors,
+ ctrl->max_hw_sectors);
+ } else {
+ ctrl->max_hw_sectors = max_hw_sectors;
+ }
+ }
+
+ kfree(id);
+ return 0;
+}
+
static void nvme_free_ctrl(struct kref *kref)
{
struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index b6c5a55..a624add7 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -62,6 +62,8 @@ struct nvme_ctrl {
u32 ctrl_config;
u32 page_size;
+ u32 max_hw_sectors;
+ u32 stripe_size;
u16 oncs;
u16 abort_limit;
u8 event_limit;
@@ -93,6 +95,7 @@ struct nvme_ns {
struct nvme_ctrl_ops {
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
+ int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
void (*free_ctrl)(struct nvme_ctrl *ctrl);
};
@@ -177,6 +180,7 @@ int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
void nvme_put_ctrl(struct nvme_ctrl *ctrl);
+int nvme_init_identify(struct nvme_ctrl *ctrl);
void nvme_put_ns(struct nvme_ns *ns);
struct request *nvme_alloc_request(struct request_queue *q,
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index ccb3151..086563f 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -129,8 +129,6 @@ struct nvme_dev {
struct work_struct probe_work;
struct work_struct scan_work;
bool subsystem;
- u32 max_hw_sectors;
- u32 stripe_size;
void __iomem *cmb;
dma_addr_t cmb_dma_addr;
u64 cmb_size;
@@ -1592,13 +1590,13 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
list_add_tail(&ns->list, &dev->namespaces);
blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
- if (dev->max_hw_sectors) {
- blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
+ if (dev->ctrl.max_hw_sectors) {
+ blk_queue_max_hw_sectors(ns->queue, dev->ctrl.max_hw_sectors);
blk_queue_max_segments(ns->queue,
- (dev->max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1);
+ (dev->ctrl.max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1);
}
- if (dev->stripe_size)
- blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
+ if (dev->ctrl.stripe_size)
+ blk_queue_chunk_sectors(ns->queue, dev->ctrl.stripe_size >> 9);
if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT)
blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
blk_queue_virt_boundary(ns->queue, dev->ctrl.page_size - 1);
@@ -1933,38 +1931,10 @@ static void nvme_dev_scan(struct work_struct *work)
static int nvme_dev_add(struct nvme_dev *dev)
{
int res;
- struct nvme_id_ctrl *ctrl;
- int shift = NVME_CAP_MPSMIN(lo_hi_readq(dev->bar + NVME_REG_CAP)) + 12;
-
- res = nvme_identify_ctrl(&dev->ctrl, &ctrl);
- if (res) {
- dev_err(dev->dev, "Identify Controller failed (%d)\n", res);
- return -EIO;
- }
-
- dev->ctrl.oncs = le16_to_cpup(&ctrl->oncs);
- dev->ctrl.abort_limit = ctrl->acl + 1;
- dev->ctrl.vwc = ctrl->vwc;
- memcpy(dev->ctrl.serial, ctrl->sn, sizeof(ctrl->sn));
- memcpy(dev->ctrl.model, ctrl->mn, sizeof(ctrl->mn));
- memcpy(dev->ctrl.firmware_rev, ctrl->fr, sizeof(ctrl->fr));
- if (ctrl->mdts)
- dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
- else
- dev->max_hw_sectors = UINT_MAX;
-
- if ((dev->ctrl.quirks & NVME_QUIRK_STRIPE_SIZE) && ctrl->vs[3]) {
- unsigned int max_hw_sectors;
- dev->stripe_size = 1 << (ctrl->vs[3] + shift);
- max_hw_sectors = dev->stripe_size >> (shift - 9);
- if (dev->max_hw_sectors) {
- dev->max_hw_sectors = min(max_hw_sectors,
- dev->max_hw_sectors);
- } else
- dev->max_hw_sectors = max_hw_sectors;
- }
- kfree(ctrl);
+ res = nvme_init_identify(&dev->ctrl);
+ if (res)
+ return res;
if (!dev->tagset.tags) {
dev->tagset.ops = &nvme_mq_ops;
@@ -2597,9 +2567,16 @@ static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
return 0;
}
+static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
+{
+ *val = readq(to_nvme_dev(ctrl)->bar + off);
+ return 0;
+}
+
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.reg_read32 = nvme_pci_reg_read32,
.reg_write32 = nvme_pci_reg_write32,
+ .reg_read64 = nvme_pci_reg_read64,
.free_ctrl = nvme_pci_free_ctrl,
};
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6/9] nvme: move the call to nvme_init_identify earlier
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
` (4 preceding siblings ...)
2015-11-30 8:40 ` [PATCH 5/9] nvme: add a common helper to read Identify Controller data Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 7/9] nvme: move namespace scanning to common code Christoph Hellwig
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
We want to record the identify and CAP values even if no I/O queue
is available.
Signed-off-by: Christoph Hellwig <hch at lst.de>
Signed-off-by: Keith Busch <keith.busch at intel.com>
---
drivers/nvme/host/pci.c | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 086563f..4d64aee 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1930,12 +1930,6 @@ static void nvme_dev_scan(struct work_struct *work)
*/
static int nvme_dev_add(struct nvme_dev *dev)
{
- int res;
-
- res = nvme_init_identify(&dev->ctrl);
- if (res)
- return res;
-
if (!dev->tagset.tags) {
dev->tagset.ops = &nvme_mq_ops;
dev->tagset.nr_hw_queues = dev->online_queues - 1;
@@ -2431,6 +2425,10 @@ static void nvme_probe_work(struct work_struct *work)
if (result)
goto disable;
+ result = nvme_init_identify(&dev->ctrl);
+ if (result)
+ goto free_tags;
+
result = nvme_setup_io_queues(dev);
if (result)
goto free_tags;
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 7/9] nvme: move namespace scanning to common code
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
` (5 preceding siblings ...)
2015-11-30 8:40 ` [PATCH 6/9] nvme: move the call to nvme_init_identify earlier Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 8/9] nvme: move chardev and sysfs interface " Christoph Hellwig
2015-11-30 8:40 ` [PATCH 9/9] nvme: refactor set_queue_count Christoph Hellwig
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
The namespace scanning code has been mostly generic already, we just
need to store a pointer to the tagset in the nvme_ctrl structure, and
add a method to check if a controller is I/O incapable. The latter
will hopefully be replaced by a proper controller state machine soon.
Signed-off-by: Christoph Hellwig <hch at lst.de>
[Fixed pr conflicts]
Signed-off-by: Keith Busch <keith.busch at intel.com>
---
drivers/nvme/host/core.c | 192 ++++++++++++++++++++++++++++++++++++++++-
drivers/nvme/host/nvme.h | 24 +++++-
drivers/nvme/host/pci.c | 220 +++++++----------------------------------------
3 files changed, 240 insertions(+), 196 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 1c9f09c..1b84984 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -18,6 +18,8 @@
#include <linux/errno.h>
#include <linux/hdreg.h>
#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/list_sort.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/pr.h>
@@ -29,6 +31,9 @@
#include "nvme.h"
+static int nvme_major;
+module_param(nvme_major, int, 0);
+
DEFINE_SPINLOCK(dev_list_lock);
static void nvme_free_ns(struct kref *kref)
@@ -47,7 +52,7 @@ static void nvme_free_ns(struct kref *kref)
kfree(ns);
}
-void nvme_put_ns(struct nvme_ns *ns)
+static void nvme_put_ns(struct nvme_ns *ns)
{
kref_put(&ns->kref, nvme_free_ns);
}
@@ -496,7 +501,7 @@ static void nvme_config_discard(struct nvme_ns *ns)
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
}
-int nvme_revalidate_disk(struct gendisk *disk)
+static int nvme_revalidate_disk(struct gendisk *disk)
{
struct nvme_ns *ns = disk->private_data;
struct nvme_id_ns *id;
@@ -660,7 +665,7 @@ static const struct pr_ops nvme_pr_ops = {
.pr_clear = nvme_pr_clear,
};
-const struct block_device_operations nvme_fops = {
+static const struct block_device_operations nvme_fops = {
.owner = THIS_MODULE,
.ioctl = nvme_ioctl,
.compat_ioctl = nvme_compat_ioctl,
@@ -840,3 +845,184 @@ void nvme_put_ctrl(struct nvme_ctrl *ctrl)
kref_put(&ctrl->kref, nvme_free_ctrl);
}
+static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
+{
+ struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
+ struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
+
+ return nsa->ns_id - nsb->ns_id;
+}
+
+static struct nvme_ns *nvme_find_ns(struct nvme_ctrl *ctrl, unsigned nsid)
+{
+ struct nvme_ns *ns;
+
+ list_for_each_entry(ns, &ctrl->namespaces, list) {
+ if (ns->ns_id == nsid)
+ return ns;
+ if (ns->ns_id > nsid)
+ break;
+ }
+ return NULL;
+}
+
+static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
+{
+ struct nvme_ns *ns;
+ struct gendisk *disk;
+ int node = dev_to_node(ctrl->dev);
+
+ ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
+ if (!ns)
+ return;
+
+ ns->queue = blk_mq_init_queue(ctrl->tagset);
+ if (IS_ERR(ns->queue))
+ goto out_free_ns;
+ queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
+ queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
+ ns->queue->queuedata = ns;
+ ns->ctrl = ctrl;
+
+ disk = alloc_disk_node(0, node);
+ if (!disk)
+ goto out_free_queue;
+
+ kref_init(&ns->kref);
+ ns->ns_id = nsid;
+ ns->disk = disk;
+ ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
+ list_add_tail(&ns->list, &ctrl->namespaces);
+
+ blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
+ if (ctrl->max_hw_sectors) {
+ blk_queue_max_hw_sectors(ns->queue, ctrl->max_hw_sectors);
+ blk_queue_max_segments(ns->queue,
+ (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1);
+ }
+ if (ctrl->stripe_size)
+ blk_queue_chunk_sectors(ns->queue, ctrl->stripe_size >> 9);
+ if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
+ blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
+ blk_queue_virt_boundary(ns->queue, ctrl->page_size - 1);
+
+ disk->major = nvme_major;
+ disk->first_minor = 0;
+ disk->fops = &nvme_fops;
+ disk->private_data = ns;
+ disk->queue = ns->queue;
+ disk->driverfs_dev = ctrl->device;
+ disk->flags = GENHD_FL_EXT_DEVT;
+ sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance, nsid);
+
+ /*
+ * Initialize capacity to 0 until we establish the namespace format and
+ * setup integrity extentions if necessary. The revalidate_disk after
+ * add_disk allows the driver to register with integrity if the format
+ * requires it.
+ */
+ set_capacity(disk, 0);
+ if (nvme_revalidate_disk(ns->disk))
+ goto out_free_disk;
+
+ kref_get(&ctrl->kref);
+ if (ns->type != NVME_NS_LIGHTNVM) {
+ add_disk(ns->disk);
+ if (ns->ms) {
+ struct block_device *bd = bdget_disk(ns->disk, 0);
+ if (!bd)
+ return;
+ if (blkdev_get(bd, FMODE_READ, NULL)) {
+ bdput(bd);
+ return;
+ }
+ blkdev_reread_part(bd);
+ blkdev_put(bd, FMODE_READ);
+ }
+ }
+
+ return;
+ out_free_disk:
+ kfree(disk);
+ list_del(&ns->list);
+ out_free_queue:
+ blk_cleanup_queue(ns->queue);
+ out_free_ns:
+ kfree(ns);
+}
+
+static void nvme_ns_remove(struct nvme_ns *ns)
+{
+ bool kill = nvme_io_incapable(ns->ctrl) &&
+ !blk_queue_dying(ns->queue);
+
+ if (kill)
+ blk_set_queue_dying(ns->queue);
+ if (ns->disk->flags & GENHD_FL_UP) {
+ if (blk_get_integrity(ns->disk))
+ blk_integrity_unregister(ns->disk);
+ del_gendisk(ns->disk);
+ }
+ if (kill || !blk_queue_dying(ns->queue)) {
+ blk_mq_abort_requeue_list(ns->queue);
+ blk_cleanup_queue(ns->queue);
+ }
+ list_del_init(&ns->list);
+ nvme_put_ns(ns);
+}
+
+static void __nvme_scan_namespaces(struct nvme_ctrl *ctrl, unsigned nn)
+{
+ struct nvme_ns *ns, *next;
+ unsigned i;
+
+ for (i = 1; i <= nn; i++) {
+ ns = nvme_find_ns(ctrl, i);
+ if (ns) {
+ if (revalidate_disk(ns->disk))
+ nvme_ns_remove(ns);
+ } else
+ nvme_alloc_ns(ctrl, i);
+ }
+ list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
+ if (ns->ns_id > nn)
+ nvme_ns_remove(ns);
+ }
+ list_sort(NULL, &ctrl->namespaces, ns_cmp);
+}
+
+void nvme_scan_namespaces(struct nvme_ctrl *ctrl)
+{
+ struct nvme_id_ctrl *id;
+
+ if (nvme_identify_ctrl(ctrl, &id))
+ return;
+ __nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn));
+ kfree(id);
+}
+
+void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
+{
+ struct nvme_ns *ns, *next;
+
+ list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
+ nvme_ns_remove(ns);
+}
+
+int __init nvme_core_init(void)
+{
+ int result;
+
+ result = register_blkdev(nvme_major, "nvme");
+ if (result < 0)
+ return result;
+ else if (result > 0)
+ nvme_major = result;
+
+ return 0;
+}
+
+void nvme_core_exit(void)
+{
+ unregister_blkdev(nvme_major, "nvme");
+}
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index a624add7..dfedaaa 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -53,6 +53,9 @@ struct nvme_ctrl {
struct device *dev;
struct kref kref;
int instance;
+ struct blk_mq_tag_set *tagset;
+ struct list_head namespaces;
+ struct device *device; /* char device */
char name[12];
char serial[20];
@@ -96,6 +99,7 @@ struct nvme_ctrl_ops {
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
+ bool (*io_incapable)(struct nvme_ctrl *ctrl);
void (*free_ctrl)(struct nvme_ctrl *ctrl);
};
@@ -108,6 +112,17 @@ static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
return val & NVME_CSTS_RDY;
}
+static inline bool nvme_io_incapable(struct nvme_ctrl *ctrl)
+{
+ u32 val = 0;
+
+ if (ctrl->ops->io_incapable(ctrl))
+ return false;
+ if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
+ return false;
+ return val & NVME_CSTS_CFS;
+}
+
static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
{
return (sector >> (ns->lba_shift - 9));
@@ -181,7 +196,9 @@ int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
void nvme_put_ctrl(struct nvme_ctrl *ctrl);
int nvme_init_identify(struct nvme_ctrl *ctrl);
-void nvme_put_ns(struct nvme_ns *ns);
+
+void nvme_scan_namespaces(struct nvme_ctrl *ctrl);
+void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
struct request *nvme_alloc_request(struct request_queue *q,
struct nvme_command *cmd, unsigned int flags);
@@ -205,10 +222,8 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
dma_addr_t dma_addr, u32 *result);
-extern const struct block_device_operations nvme_fops;
extern spinlock_t dev_list_lock;
-int nvme_revalidate_disk(struct gendisk *disk);
int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
struct nvme_passthru_cmd __user *ucmd);
@@ -222,4 +237,7 @@ int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id);
int nvme_nvm_register(struct request_queue *q, char *disk_name);
void nvme_nvm_unregister(struct request_queue *q, char *disk_name);
+int __init nvme_core_init(void);
+void nvme_core_exit(void);
+
#endif /* _NVME_H */
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 4d64aee..697dc1f 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -28,7 +28,6 @@
#include <linux/kdev_t.h>
#include <linux/kthread.h>
#include <linux/kernel.h>
-#include <linux/list_sort.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
@@ -65,9 +64,6 @@ unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
-static int nvme_major;
-module_param(nvme_major, int, 0);
-
static int nvme_char_major;
module_param(nvme_char_major, int, 0);
@@ -123,8 +119,6 @@ struct nvme_dev {
u32 db_stride;
struct msix_entry *entry;
void __iomem *bar;
- struct list_head namespaces;
- struct device *device;
struct work_struct reset_work;
struct work_struct probe_work;
struct work_struct scan_work;
@@ -1561,90 +1555,6 @@ static int nvme_kthread(void *data)
return 0;
}
-static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
-{
- struct nvme_ns *ns;
- struct gendisk *disk;
- int node = dev_to_node(dev->dev);
-
- ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
- if (!ns)
- return;
-
- ns->queue = blk_mq_init_queue(&dev->tagset);
- if (IS_ERR(ns->queue))
- goto out_free_ns;
- queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
- queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
- ns->ctrl = &dev->ctrl;
- ns->queue->queuedata = ns;
-
- disk = alloc_disk_node(0, node);
- if (!disk)
- goto out_free_queue;
-
- kref_init(&ns->kref);
- ns->ns_id = nsid;
- ns->disk = disk;
- ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
- list_add_tail(&ns->list, &dev->namespaces);
-
- blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
- if (dev->ctrl.max_hw_sectors) {
- blk_queue_max_hw_sectors(ns->queue, dev->ctrl.max_hw_sectors);
- blk_queue_max_segments(ns->queue,
- (dev->ctrl.max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1);
- }
- if (dev->ctrl.stripe_size)
- blk_queue_chunk_sectors(ns->queue, dev->ctrl.stripe_size >> 9);
- if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT)
- blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
- blk_queue_virt_boundary(ns->queue, dev->ctrl.page_size - 1);
-
- disk->major = nvme_major;
- disk->first_minor = 0;
- disk->fops = &nvme_fops;
- disk->private_data = ns;
- disk->queue = ns->queue;
- disk->driverfs_dev = dev->device;
- disk->flags = GENHD_FL_EXT_DEVT;
- sprintf(disk->disk_name, "nvme%dn%d", dev->ctrl.instance, nsid);
-
- /*
- * Initialize capacity to 0 until we establish the namespace format and
- * setup integrity extentions if necessary. The revalidate_disk after
- * add_disk allows the driver to register with integrity if the format
- * requires it.
- */
- set_capacity(disk, 0);
- if (nvme_revalidate_disk(ns->disk))
- goto out_free_disk;
-
- kref_get(&dev->ctrl.kref);
- if (ns->type != NVME_NS_LIGHTNVM) {
- add_disk(ns->disk);
- if (ns->ms) {
- struct block_device *bd = bdget_disk(ns->disk, 0);
- if (!bd)
- return;
- if (blkdev_get(bd, FMODE_READ, NULL)) {
- bdput(bd);
- return;
- }
- blkdev_reread_part(bd);
- blkdev_put(bd, FMODE_READ);
- }
- }
- return;
- out_free_disk:
- kfree(disk);
- list_del(&ns->list);
- out_free_queue:
- blk_cleanup_queue(ns->queue);
- out_free_ns:
- kfree(ns);
-}
-
/*
* Create I/O queues. Failing to create an I/O queue is not an issue,
* we can continue with less than the desired amount of queues, and
@@ -1827,71 +1737,6 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
return result;
}
-static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
-{
- struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
- struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
-
- return nsa->ns_id - nsb->ns_id;
-}
-
-static struct nvme_ns *nvme_find_ns(struct nvme_dev *dev, unsigned nsid)
-{
- struct nvme_ns *ns;
-
- list_for_each_entry(ns, &dev->namespaces, list) {
- if (ns->ns_id == nsid)
- return ns;
- if (ns->ns_id > nsid)
- break;
- }
- return NULL;
-}
-
-static inline bool nvme_io_incapable(struct nvme_dev *dev)
-{
- return (!dev->bar ||
- readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_CFS ||
- dev->online_queues < 2);
-}
-
-static void nvme_ns_remove(struct nvme_ns *ns)
-{
- bool kill = nvme_io_incapable(to_nvme_dev(ns->ctrl)) &&
- !blk_queue_dying(ns->queue);
-
- if (kill)
- blk_set_queue_dying(ns->queue);
- if (ns->disk->flags & GENHD_FL_UP)
- del_gendisk(ns->disk);
- if (kill || !blk_queue_dying(ns->queue)) {
- blk_mq_abort_requeue_list(ns->queue);
- blk_cleanup_queue(ns->queue);
- }
- list_del_init(&ns->list);
- nvme_put_ns(ns);
-}
-
-static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
-{
- struct nvme_ns *ns, *next;
- unsigned i;
-
- for (i = 1; i <= nn; i++) {
- ns = nvme_find_ns(dev, i);
- if (ns) {
- if (revalidate_disk(ns->disk))
- nvme_ns_remove(ns);
- } else
- nvme_alloc_ns(dev, i);
- }
- list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
- if (ns->ns_id > nn)
- nvme_ns_remove(ns);
- }
- list_sort(NULL, &dev->namespaces, ns_cmp);
-}
-
static void nvme_set_irq_hints(struct nvme_dev *dev)
{
struct nvme_queue *nvmeq;
@@ -1911,14 +1756,10 @@ static void nvme_set_irq_hints(struct nvme_dev *dev)
static void nvme_dev_scan(struct work_struct *work)
{
struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
- struct nvme_id_ctrl *ctrl;
if (!dev->tagset.tags)
return;
- if (nvme_identify_ctrl(&dev->ctrl, &ctrl))
- return;
- nvme_scan_namespaces(dev, le32_to_cpup(&ctrl->nn));
- kfree(ctrl);
+ nvme_scan_namespaces(&dev->ctrl);
nvme_set_irq_hints(dev);
}
@@ -1930,7 +1771,7 @@ static void nvme_dev_scan(struct work_struct *work)
*/
static int nvme_dev_add(struct nvme_dev *dev)
{
- if (!dev->tagset.tags) {
+ if (!dev->ctrl.tagset) {
dev->tagset.ops = &nvme_mq_ops;
dev->tagset.nr_hw_queues = dev->online_queues - 1;
dev->tagset.timeout = NVME_IO_TIMEOUT;
@@ -1943,6 +1784,7 @@ static int nvme_dev_add(struct nvme_dev *dev)
if (blk_mq_alloc_tag_set(&dev->tagset))
return 0;
+ dev->ctrl.tagset = &dev->tagset;
}
schedule_work(&dev->scan_work);
return 0;
@@ -2197,7 +2039,7 @@ static void nvme_freeze_queues(struct nvme_dev *dev)
{
struct nvme_ns *ns;
- list_for_each_entry(ns, &dev->namespaces, list) {
+ list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
blk_mq_freeze_queue_start(ns->queue);
spin_lock_irq(ns->queue->queue_lock);
@@ -2213,7 +2055,7 @@ static void nvme_unfreeze_queues(struct nvme_dev *dev)
{
struct nvme_ns *ns;
- list_for_each_entry(ns, &dev->namespaces, list) {
+ list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
blk_mq_unfreeze_queue(ns->queue);
blk_mq_start_stopped_hw_queues(ns->queue, true);
@@ -2248,14 +2090,6 @@ static void nvme_dev_shutdown(struct nvme_dev *dev)
nvme_clear_queue(dev->queues[i]);
}
-static void nvme_dev_remove(struct nvme_dev *dev)
-{
- struct nvme_ns *ns, *next;
-
- list_for_each_entry_safe(ns, next, &dev->namespaces, list)
- nvme_ns_remove(ns);
-}
-
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
@@ -2313,7 +2147,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
struct nvme_dev *dev = to_nvme_dev(ctrl);
put_device(dev->dev);
- put_device(dev->device);
+ put_device(ctrl->device);
nvme_release_instance(dev);
if (dev->tagset.tags)
blk_mq_free_tag_set(&dev->tagset);
@@ -2365,9 +2199,9 @@ static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
case NVME_IOCTL_ADMIN_CMD:
return nvme_user_cmd(&dev->ctrl, NULL, (void __user *)arg);
case NVME_IOCTL_IO_CMD:
- if (list_empty(&dev->namespaces))
+ if (list_empty(&dev->ctrl.namespaces))
return -ENOTTY;
- ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
+ ns = list_first_entry(&dev->ctrl.namespaces, struct nvme_ns, list);
return nvme_user_cmd(&dev->ctrl, ns, (void __user *)arg);
case NVME_IOCTL_RESET:
dev_warn(dev->dev, "resetting controller\n");
@@ -2441,7 +2275,7 @@ static void nvme_probe_work(struct work_struct *work)
*/
if (dev->online_queues < 2) {
dev_warn(dev->dev, "IO queues not created\n");
- nvme_dev_remove(dev);
+ nvme_remove_namespaces(&dev->ctrl);
} else {
nvme_unfreeze_queues(dev);
nvme_dev_add(dev);
@@ -2571,10 +2405,18 @@ static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
return 0;
}
+static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
+{
+ struct nvme_dev *dev = to_nvme_dev(ctrl);
+
+ return !dev->bar || dev->online_queues < 2;
+}
+
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.reg_read32 = nvme_pci_reg_read32,
.reg_write32 = nvme_pci_reg_write32,
.reg_read64 = nvme_pci_reg_read64,
+ .io_incapable = nvme_pci_io_incapable,
.free_ctrl = nvme_pci_free_ctrl,
};
@@ -2599,7 +2441,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (!dev->queues)
goto free;
- INIT_LIST_HEAD(&dev->namespaces);
+ INIT_LIST_HEAD(&dev->ctrl.namespaces);
INIT_WORK(&dev->reset_work, nvme_reset_work);
dev->dev = get_device(&pdev->dev);
pci_set_drvdata(pdev, dev);
@@ -2617,17 +2459,17 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
goto release;
kref_init(&dev->ctrl.kref);
- dev->device = device_create(nvme_class, &pdev->dev,
+ dev->ctrl.device = device_create(nvme_class, &pdev->dev,
MKDEV(nvme_char_major, dev->ctrl.instance),
dev, "nvme%d", dev->ctrl.instance);
- if (IS_ERR(dev->device)) {
- result = PTR_ERR(dev->device);
+ if (IS_ERR(dev->ctrl.device)) {
+ result = PTR_ERR(dev->ctrl.device);
goto release_pools;
}
- get_device(dev->device);
- dev_set_drvdata(dev->device, dev);
+ get_device(dev->ctrl.device);
+ dev_set_drvdata(dev->ctrl.device, dev);
- result = device_create_file(dev->device, &dev_attr_reset_controller);
+ result = device_create_file(dev->ctrl.device, &dev_attr_reset_controller);
if (result)
goto put_dev;
@@ -2639,7 +2481,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
put_dev:
device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
- put_device(dev->device);
+ put_device(dev->ctrl.device);
release_pools:
nvme_release_prp_pools(dev);
release:
@@ -2681,8 +2523,8 @@ static void nvme_remove(struct pci_dev *pdev)
flush_work(&dev->probe_work);
flush_work(&dev->reset_work);
flush_work(&dev->scan_work);
- device_remove_file(dev->device, &dev_attr_reset_controller);
- nvme_dev_remove(dev);
+ device_remove_file(dev->ctrl.device, &dev_attr_reset_controller);
+ nvme_remove_namespaces(&dev->ctrl);
nvme_dev_shutdown(dev);
nvme_dev_remove_admin(dev);
device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
@@ -2764,11 +2606,9 @@ static int __init nvme_init(void)
if (!nvme_workq)
return -ENOMEM;
- result = register_blkdev(nvme_major, "nvme");
+ result = nvme_core_init();
if (result < 0)
goto kill_workq;
- else if (result > 0)
- nvme_major = result;
result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
&nvme_dev_fops);
@@ -2793,7 +2633,7 @@ static int __init nvme_init(void)
unregister_chrdev:
__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
unregister_blkdev:
- unregister_blkdev(nvme_major, "nvme");
+ nvme_core_exit();
kill_workq:
destroy_workqueue(nvme_workq);
return result;
@@ -2802,7 +2642,7 @@ static int __init nvme_init(void)
static void __exit nvme_exit(void)
{
pci_unregister_driver(&nvme_driver);
- unregister_blkdev(nvme_major, "nvme");
+ nvme_core_exit();
destroy_workqueue(nvme_workq);
class_destroy(nvme_class);
__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 8/9] nvme: move chardev and sysfs interface to common code
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
` (6 preceding siblings ...)
2015-11-30 8:40 ` [PATCH 7/9] nvme: move namespace scanning to common code Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
2015-11-30 8:40 ` [PATCH 9/9] nvme: refactor set_queue_count Christoph Hellwig
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
For this we need to add a proper controller init routine and a list of
all controllers that is in addition to the list of PCIe controllers,
which stays in pci.c. Note that we remove the sysfs device when the
last reference to a controller is dropped now - the old code would have
kept it around longer, which doesn't make much sense.
This requires a new ->reset_ctrl operation to implement controleller
resets, and a new ->write_reg32 operation that is required to implement
subsystem resets. We also now store caches copied of the NVMe compliance
version and the flag if a controller is attached to a subsystem or not in
the generic controller structure now.
Signed-off-by: Christoph Hellwig <hch at lst.de>
[Fixes for pr merge]
Signed-off-by: Keith Busch <keith.busch at intel.com>
---
drivers/nvme/host/core.c | 215 +++++++++++++++++++++++++++++++++++++++++++++--
drivers/nvme/host/nvme.h | 18 ++--
drivers/nvme/host/pci.c | 195 ++++--------------------------------------
drivers/nvme/host/scsi.c | 13 +--
4 files changed, 241 insertions(+), 200 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 1b84984..9c7dfd1 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -31,11 +31,19 @@
#include "nvme.h"
+#define NVME_MINORS (1U << MINORBITS)
+
static int nvme_major;
module_param(nvme_major, int, 0);
+static int nvme_char_major;
+module_param(nvme_char_major, int, 0);
+
+static LIST_HEAD(nvme_ctrl_list);
DEFINE_SPINLOCK(dev_list_lock);
+static struct class *nvme_class;
+
static void nvme_free_ns(struct kref *kref)
{
struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
@@ -367,7 +375,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
metadata, meta_len, io.slba, NULL, 0);
}
-int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
+static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
struct nvme_passthru_cmd __user *ucmd)
{
struct nvme_passthru_cmd cmd;
@@ -792,6 +800,12 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
u64 cap;
int ret, page_shift;
+ ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
+ if (ret) {
+ dev_err(ctrl->dev, "Reading VS failed (%d)\n", ret);
+ return ret;
+ }
+
ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
if (ret) {
dev_err(ctrl->dev, "Reading CAP failed (%d)\n", ret);
@@ -799,6 +813,9 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
}
page_shift = NVME_CAP_MPSMIN(cap) + 12;
+ if (ctrl->vs >= NVME_VS(1, 1))
+ ctrl->subsystem = NVME_CAP_NSSRC(cap);
+
ret = nvme_identify_ctrl(ctrl, &id);
if (ret) {
dev_err(ctrl->dev, "Identify Controller failed (%d)\n", ret);
@@ -833,17 +850,84 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
return 0;
}
-static void nvme_free_ctrl(struct kref *kref)
+static int nvme_dev_open(struct inode *inode, struct file *file)
{
- struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
+ struct nvme_ctrl *ctrl;
+ int instance = iminor(inode);
+ int ret = -ENODEV;
- ctrl->ops->free_ctrl(ctrl);
+ spin_lock(&dev_list_lock);
+ list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
+ if (ctrl->instance != instance)
+ continue;
+
+ if (!ctrl->admin_q) {
+ ret = -EWOULDBLOCK;
+ break;
+ }
+ if (!kref_get_unless_zero(&ctrl->kref))
+ break;
+ file->private_data = ctrl;
+ ret = 0;
+ break;
+ }
+ spin_unlock(&dev_list_lock);
+
+ return ret;
}
-void nvme_put_ctrl(struct nvme_ctrl *ctrl)
+static int nvme_dev_release(struct inode *inode, struct file *file)
{
- kref_put(&ctrl->kref, nvme_free_ctrl);
+ nvme_put_ctrl(file->private_data);
+ return 0;
+}
+
+static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ struct nvme_ctrl *ctrl = file->private_data;
+ void __user *argp = (void __user *)arg;
+ struct nvme_ns *ns;
+
+ switch (cmd) {
+ case NVME_IOCTL_ADMIN_CMD:
+ return nvme_user_cmd(ctrl, NULL, argp);
+ case NVME_IOCTL_IO_CMD:
+ if (list_empty(&ctrl->namespaces))
+ return -ENOTTY;
+ ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
+ return nvme_user_cmd(ctrl, ns, argp);
+ case NVME_IOCTL_RESET:
+ dev_warn(ctrl->dev, "resetting controller\n");
+ return ctrl->ops->reset_ctrl(ctrl);
+ case NVME_IOCTL_SUBSYS_RESET:
+ return nvme_reset_subsystem(ctrl);
+ default:
+ return -ENOTTY;
+ }
+}
+
+static const struct file_operations nvme_dev_fops = {
+ .owner = THIS_MODULE,
+ .open = nvme_dev_open,
+ .release = nvme_dev_release,
+ .unlocked_ioctl = nvme_dev_ioctl,
+ .compat_ioctl = nvme_dev_ioctl,
+};
+
+static ssize_t nvme_sysfs_reset(struct device *dev,
+ struct device_attribute *attr, const char *buf,
+ size_t count)
+{
+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
+ int ret;
+
+ ret = ctrl->ops->reset_ctrl(ctrl);
+ if (ret < 0)
+ return ret;
+ return count;
}
+static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
@@ -1009,6 +1093,104 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
nvme_ns_remove(ns);
}
+static DEFINE_IDA(nvme_instance_ida);
+
+static int nvme_set_instance(struct nvme_ctrl *ctrl)
+{
+ int instance, error;
+
+ do {
+ if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
+ return -ENODEV;
+
+ spin_lock(&dev_list_lock);
+ error = ida_get_new(&nvme_instance_ida, &instance);
+ spin_unlock(&dev_list_lock);
+ } while (error == -EAGAIN);
+
+ if (error)
+ return -ENODEV;
+
+ ctrl->instance = instance;
+ return 0;
+}
+
+static void nvme_release_instance(struct nvme_ctrl *ctrl)
+{
+ spin_lock(&dev_list_lock);
+ ida_remove(&nvme_instance_ida, ctrl->instance);
+ spin_unlock(&dev_list_lock);
+}
+
+static void nvme_free_ctrl(struct kref *kref)
+{
+ struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
+
+ spin_lock(&dev_list_lock);
+ list_del(&ctrl->node);
+ spin_unlock(&dev_list_lock);
+
+ put_device(ctrl->device);
+ nvme_release_instance(ctrl);
+ device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
+
+ ctrl->ops->free_ctrl(ctrl);
+}
+
+void nvme_put_ctrl(struct nvme_ctrl *ctrl)
+{
+ kref_put(&ctrl->kref, nvme_free_ctrl);
+}
+
+/*
+ * Initialize a NVMe controller structures. This needs to be called during
+ * earliest initialization so that we have the initialized structured around
+ * during probing.
+ */
+int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
+ const struct nvme_ctrl_ops *ops, unsigned long quirks)
+{
+ int ret;
+
+ INIT_LIST_HEAD(&ctrl->namespaces);
+ kref_init(&ctrl->kref);
+ ctrl->dev = dev;
+ ctrl->ops = ops;
+ ctrl->quirks = quirks;
+
+ ret = nvme_set_instance(ctrl);
+ if (ret)
+ goto out;
+
+ ctrl->device = device_create(nvme_class, ctrl->dev,
+ MKDEV(nvme_char_major, ctrl->instance),
+ dev, "nvme%d", ctrl->instance);
+ if (IS_ERR(ctrl->device)) {
+ ret = PTR_ERR(ctrl->device);
+ goto out_release_instance;
+ }
+ get_device(ctrl->device);
+ dev_set_drvdata(ctrl->device, ctrl);
+
+ ret = device_create_file(ctrl->device, &dev_attr_reset_controller);
+ if (ret)
+ goto out_put_device;
+
+ spin_lock(&dev_list_lock);
+ list_add_tail(&ctrl->node, &nvme_ctrl_list);
+ spin_unlock(&dev_list_lock);
+
+ return 0;
+
+out_put_device:
+ put_device(ctrl->device);
+ device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
+out_release_instance:
+ nvme_release_instance(ctrl);
+out:
+ return ret;
+}
+
int __init nvme_core_init(void)
{
int result;
@@ -1019,10 +1201,31 @@ int __init nvme_core_init(void)
else if (result > 0)
nvme_major = result;
+ result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
+ &nvme_dev_fops);
+ if (result < 0)
+ goto unregister_blkdev;
+ else if (result > 0)
+ nvme_char_major = result;
+
+ nvme_class = class_create(THIS_MODULE, "nvme");
+ if (IS_ERR(nvme_class)) {
+ result = PTR_ERR(nvme_class);
+ goto unregister_chrdev;
+ }
+
return 0;
+
+ unregister_chrdev:
+ __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
+ unregister_blkdev:
+ unregister_blkdev(nvme_major, "nvme");
+ return result;
}
void nvme_core_exit(void)
{
unregister_blkdev(nvme_major, "nvme");
+ class_destroy(nvme_class);
+ __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
}
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index dfedaaa..93378be 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -19,8 +19,6 @@
#include <linux/kref.h>
#include <linux/blk-mq.h>
-struct nvme_passthru_cmd;
-
extern unsigned char nvme_io_timeout;
#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
@@ -56,6 +54,7 @@ struct nvme_ctrl {
struct blk_mq_tag_set *tagset;
struct list_head namespaces;
struct device *device; /* char device */
+ struct list_head node;
char name[12];
char serial[20];
@@ -71,6 +70,8 @@ struct nvme_ctrl {
u16 abort_limit;
u8 event_limit;
u8 vwc;
+ u32 vs;
+ bool subsystem;
unsigned long quirks;
};
@@ -100,6 +101,7 @@ struct nvme_ctrl_ops {
int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
bool (*io_incapable)(struct nvme_ctrl *ctrl);
+ int (*reset_ctrl)(struct nvme_ctrl *ctrl);
void (*free_ctrl)(struct nvme_ctrl *ctrl);
};
@@ -123,6 +125,13 @@ static inline bool nvme_io_incapable(struct nvme_ctrl *ctrl)
return val & NVME_CSTS_CFS;
}
+static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
+{
+ if (!ctrl->subsystem)
+ return -ENOTTY;
+ return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
+}
+
static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
{
return (sector >> (ns->lba_shift - 9));
@@ -194,6 +203,8 @@ static inline int nvme_error_status(u16 status)
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
+int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
+ const struct nvme_ctrl_ops *ops, unsigned long quirks);
void nvme_put_ctrl(struct nvme_ctrl *ctrl);
int nvme_init_identify(struct nvme_ctrl *ctrl);
@@ -224,9 +235,6 @@ int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
extern spinlock_t dev_list_lock;
-int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
- struct nvme_passthru_cmd __user *ucmd);
-
struct sg_io_hdr;
int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 697dc1f..87ad57bc 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -38,15 +38,11 @@
#include <linux/slab.h>
#include <linux/t10-pi.h>
#include <linux/types.h>
-#include <linux/pr.h>
-#include <scsi/sg.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <asm/unaligned.h>
-#include <uapi/linux/nvme_ioctl.h>
#include "nvme.h"
-#define NVME_MINORS (1U << MINORBITS)
#define NVME_Q_DEPTH 1024
#define NVME_AQ_DEPTH 256
#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
@@ -64,9 +60,6 @@ unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
-static int nvme_char_major;
-module_param(nvme_char_major, int, 0);
-
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);
@@ -79,8 +72,6 @@ static struct task_struct *nvme_thread;
static struct workqueue_struct *nvme_workq;
static wait_queue_head_t nvme_kthread_wait;
-static struct class *nvme_class;
-
struct nvme_dev;
struct nvme_queue;
struct nvme_iod;
@@ -1505,15 +1496,6 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
return result;
}
-static int nvme_subsys_reset(struct nvme_dev *dev)
-{
- if (!dev->subsystem)
- return -ENOTTY;
-
- writel(0x4E564D65, dev->bar + NVME_REG_NSSR); /* "NVMe" */
- return 0;
-}
-
static int nvme_kthread(void *data)
{
struct nvme_dev *dev, *next;
@@ -2113,42 +2095,11 @@ static void nvme_release_prp_pools(struct nvme_dev *dev)
dma_pool_destroy(dev->prp_small_pool);
}
-static DEFINE_IDA(nvme_instance_ida);
-
-static int nvme_set_instance(struct nvme_dev *dev)
-{
- int instance, error;
-
- do {
- if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
- return -ENODEV;
-
- spin_lock(&dev_list_lock);
- error = ida_get_new(&nvme_instance_ida, &instance);
- spin_unlock(&dev_list_lock);
- } while (error == -EAGAIN);
-
- if (error)
- return -ENODEV;
-
- dev->ctrl.instance = instance;
- return 0;
-}
-
-static void nvme_release_instance(struct nvme_dev *dev)
-{
- spin_lock(&dev_list_lock);
- ida_remove(&nvme_instance_ida, dev->ctrl.instance);
- spin_unlock(&dev_list_lock);
-}
-
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
{
struct nvme_dev *dev = to_nvme_dev(ctrl);
put_device(dev->dev);
- put_device(ctrl->device);
- nvme_release_instance(dev);
if (dev->tagset.tags)
blk_mq_free_tag_set(&dev->tagset);
if (dev->ctrl.admin_q)
@@ -2158,69 +2109,6 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
kfree(dev);
}
-static int nvme_dev_open(struct inode *inode, struct file *f)
-{
- struct nvme_dev *dev;
- int instance = iminor(inode);
- int ret = -ENODEV;
-
- spin_lock(&dev_list_lock);
- list_for_each_entry(dev, &dev_list, node) {
- if (dev->ctrl.instance == instance) {
- if (!dev->ctrl.admin_q) {
- ret = -EWOULDBLOCK;
- break;
- }
- if (!kref_get_unless_zero(&dev->ctrl.kref))
- break;
- f->private_data = dev;
- ret = 0;
- break;
- }
- }
- spin_unlock(&dev_list_lock);
-
- return ret;
-}
-
-static int nvme_dev_release(struct inode *inode, struct file *f)
-{
- struct nvme_dev *dev = f->private_data;
- nvme_put_ctrl(&dev->ctrl);
- return 0;
-}
-
-static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
-{
- struct nvme_dev *dev = f->private_data;
- struct nvme_ns *ns;
-
- switch (cmd) {
- case NVME_IOCTL_ADMIN_CMD:
- return nvme_user_cmd(&dev->ctrl, NULL, (void __user *)arg);
- case NVME_IOCTL_IO_CMD:
- if (list_empty(&dev->ctrl.namespaces))
- return -ENOTTY;
- ns = list_first_entry(&dev->ctrl.namespaces, struct nvme_ns, list);
- return nvme_user_cmd(&dev->ctrl, ns, (void __user *)arg);
- case NVME_IOCTL_RESET:
- dev_warn(dev->dev, "resetting controller\n");
- return nvme_reset(dev);
- case NVME_IOCTL_SUBSYS_RESET:
- return nvme_subsys_reset(dev);
- default:
- return -ENOTTY;
- }
-}
-
-static const struct file_operations nvme_dev_fops = {
- .owner = THIS_MODULE,
- .open = nvme_dev_open,
- .release = nvme_dev_release,
- .unlocked_ioctl = nvme_dev_ioctl,
- .compat_ioctl = nvme_dev_ioctl,
-};
-
static void nvme_probe_work(struct work_struct *work)
{
struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
@@ -2372,21 +2260,6 @@ static int nvme_reset(struct nvme_dev *dev)
return ret;
}
-static ssize_t nvme_sysfs_reset(struct device *dev,
- struct device_attribute *attr, const char *buf,
- size_t count)
-{
- struct nvme_dev *ndev = dev_get_drvdata(dev);
- int ret;
-
- ret = nvme_reset(ndev);
- if (ret < 0)
- return ret;
-
- return count;
-}
-static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
-
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
{
*val = readl(to_nvme_dev(ctrl)->bar + off);
@@ -2412,11 +2285,17 @@ static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
return !dev->bar || dev->online_queues < 2;
}
+static int nvme_pci_reset_ctrl(struct nvme_ctrl *ctrl)
+{
+ return nvme_reset(to_nvme_dev(ctrl));
+}
+
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.reg_read32 = nvme_pci_reg_read32,
.reg_write32 = nvme_pci_reg_write32,
.reg_read64 = nvme_pci_reg_read64,
.io_incapable = nvme_pci_io_incapable,
+ .reset_ctrl = nvme_pci_reset_ctrl,
.free_ctrl = nvme_pci_free_ctrl,
};
@@ -2441,51 +2320,28 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (!dev->queues)
goto free;
- INIT_LIST_HEAD(&dev->ctrl.namespaces);
- INIT_WORK(&dev->reset_work, nvme_reset_work);
dev->dev = get_device(&pdev->dev);
pci_set_drvdata(pdev, dev);
- dev->ctrl.ops = &nvme_pci_ctrl_ops;
- dev->ctrl.dev = dev->dev;
- dev->ctrl.quirks = id->driver_data;
+ INIT_LIST_HEAD(&dev->node);
+ INIT_WORK(&dev->scan_work, nvme_dev_scan);
+ INIT_WORK(&dev->probe_work, nvme_probe_work);
+ INIT_WORK(&dev->reset_work, nvme_reset_work);
- result = nvme_set_instance(dev);
+ result = nvme_setup_prp_pools(dev);
if (result)
goto put_pci;
- result = nvme_setup_prp_pools(dev);
+ result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
+ id->driver_data);
if (result)
- goto release;
-
- kref_init(&dev->ctrl.kref);
- dev->ctrl.device = device_create(nvme_class, &pdev->dev,
- MKDEV(nvme_char_major, dev->ctrl.instance),
- dev, "nvme%d", dev->ctrl.instance);
- if (IS_ERR(dev->ctrl.device)) {
- result = PTR_ERR(dev->ctrl.device);
goto release_pools;
- }
- get_device(dev->ctrl.device);
- dev_set_drvdata(dev->ctrl.device, dev);
-
- result = device_create_file(dev->ctrl.device, &dev_attr_reset_controller);
- if (result)
- goto put_dev;
- INIT_LIST_HEAD(&dev->node);
- INIT_WORK(&dev->scan_work, nvme_dev_scan);
- INIT_WORK(&dev->probe_work, nvme_probe_work);
schedule_work(&dev->probe_work);
return 0;
- put_dev:
- device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
- put_device(dev->ctrl.device);
release_pools:
nvme_release_prp_pools(dev);
- release:
- nvme_release_instance(dev);
put_pci:
put_device(dev->dev);
free:
@@ -2523,11 +2379,9 @@ static void nvme_remove(struct pci_dev *pdev)
flush_work(&dev->probe_work);
flush_work(&dev->reset_work);
flush_work(&dev->scan_work);
- device_remove_file(dev->ctrl.device, &dev_attr_reset_controller);
nvme_remove_namespaces(&dev->ctrl);
nvme_dev_shutdown(dev);
nvme_dev_remove_admin(dev);
- device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
nvme_free_queues(dev, 0);
nvme_release_cmb(dev);
nvme_release_prp_pools(dev);
@@ -2610,29 +2464,12 @@ static int __init nvme_init(void)
if (result < 0)
goto kill_workq;
- result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
- &nvme_dev_fops);
- if (result < 0)
- goto unregister_blkdev;
- else if (result > 0)
- nvme_char_major = result;
-
- nvme_class = class_create(THIS_MODULE, "nvme");
- if (IS_ERR(nvme_class)) {
- result = PTR_ERR(nvme_class);
- goto unregister_chrdev;
- }
-
result = pci_register_driver(&nvme_driver);
if (result)
- goto destroy_class;
+ goto core_exit;
return 0;
- destroy_class:
- class_destroy(nvme_class);
- unregister_chrdev:
- __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
- unregister_blkdev:
+ core_exit:
nvme_core_exit();
kill_workq:
destroy_workqueue(nvme_workq);
@@ -2644,8 +2481,6 @@ static void __exit nvme_exit(void)
pci_unregister_driver(&nvme_driver);
nvme_core_exit();
destroy_workqueue(nvme_workq);
- class_destroy(nvme_class);
- __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
_nvme_check_size();
}
diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c
index eaf7256..e947e29 100644
--- a/drivers/nvme/host/scsi.c
+++ b/drivers/nvme/host/scsi.c
@@ -600,7 +600,7 @@ static int nvme_trans_unit_serial_page(struct nvme_ns *ns,
}
static int nvme_fill_device_id_eui64(struct nvme_ns *ns, struct sg_io_hdr *hdr,
- u8 *inq_response, int alloc_len, u32 vs)
+ u8 *inq_response, int alloc_len)
{
struct nvme_id_ns *id_ns;
int nvme_sc, res;
@@ -615,7 +615,7 @@ static int nvme_fill_device_id_eui64(struct nvme_ns *ns, struct sg_io_hdr *hdr,
eui = id_ns->eui64;
len = sizeof(id_ns->eui64);
- if (vs >= NVME_VS(1, 2)) {
+ if (ns->ctrl->vs >= NVME_VS(1, 2)) {
if (bitmap_empty(eui, len * 8)) {
eui = id_ns->nguid;
len = sizeof(id_ns->nguid);
@@ -687,14 +687,9 @@ static int nvme_trans_device_id_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
u8 *resp, int alloc_len)
{
int res;
- u32 vs;
- res = ns->ctrl->ops->reg_read32(ns->ctrl, NVME_REG_VS, &vs);
- if (res)
- return res;
-
- if (vs >= NVME_VS(1, 1)) {
- res = nvme_fill_device_id_eui64(ns, hdr, resp, alloc_len, vs);
+ if (ns->ctrl->vs >= NVME_VS(1, 1)) {
+ res = nvme_fill_device_id_eui64(ns, hdr, resp, alloc_len);
if (res != -EOPNOTSUPP)
return res;
}
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 9/9] nvme: refactor set_queue_count
2015-11-30 8:40 NVMe driver split for Linux 4.5, part2 Christoph Hellwig
` (7 preceding siblings ...)
2015-11-30 8:40 ` [PATCH 8/9] nvme: move chardev and sysfs interface " Christoph Hellwig
@ 2015-11-30 8:40 ` Christoph Hellwig
8 siblings, 0 replies; 10+ messages in thread
From: Christoph Hellwig @ 2015-11-30 8:40 UTC (permalink / raw)
Split out a helper that just issues the Set Features and interprets the
result which can go to common code, and document why we are ignoring
non-timeout error returns in the PCIe driver.
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
drivers/nvme/host/core.c | 16 ++++++++++++++++
drivers/nvme/host/nvme.h | 1 +
drivers/nvme/host/pci.c | 34 +++++++++++++---------------------
3 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 9c7dfd1..c61bde9 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -327,6 +327,22 @@ int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
return error;
}
+int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
+{
+ u32 q_count = (*count - 1) | ((*count - 1) << 16);
+ u32 result;
+ int status, nr_io_queues;
+
+ status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
+ &result);
+ if (status)
+ return status;
+
+ nr_io_queues = min(result & 0xffff, result >> 16) + 1;
+ *count = min(*count, nr_io_queues);
+ return 0;
+}
+
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
struct nvme_user_io io;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 93378be..b75d41e 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -232,6 +232,7 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
dma_addr_t dma_addr, u32 *result);
int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
dma_addr_t dma_addr, u32 *result);
+int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
extern spinlock_t dev_list_lock;
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 87ad57bc..a64d0ba 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1559,23 +1559,6 @@ static void nvme_create_io_queues(struct nvme_dev *dev)
}
}
-static int set_queue_count(struct nvme_dev *dev, int count)
-{
- int status;
- u32 result;
- u32 q_count = (count - 1) | ((count - 1) << 16);
-
- status = nvme_set_features(&dev->ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
- &result);
- if (status < 0)
- return status;
- if (status > 0) {
- dev_err(dev->dev, "Could not set queue count (%d)\n", status);
- return 0;
- }
- return min(result & 0xffff, result >> 16) + 1;
-}
-
static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
u64 szu, size, offset;
@@ -1640,11 +1623,20 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
int result, i, vecs, nr_io_queues, size;
nr_io_queues = num_possible_cpus();
- result = set_queue_count(dev, nr_io_queues);
- if (result <= 0)
+ result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
+ if (result < 0)
return result;
- if (result < nr_io_queues)
- nr_io_queues = result;
+
+ /*
+ * Degraded controllers might return an error when setting the queue
+ * count. We still want to be able to bring them online and offer
+ * access to the admin queue, as that might be only way to fix them up.
+ */
+ if (result > 0) {
+ dev_err(dev->dev, "Could not set queue count (%d)\n", result);
+ nr_io_queues = 0;
+ result = 0;
+ }
if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
result = nvme_cmb_qdepth(dev, nr_io_queues,
--
1.9.1
^ permalink raw reply related [flat|nested] 10+ messages in thread