From: John Meneghini <jmeneghi@redhat.com>
To: tj@kernel.org, josef@toxicpanda.com, axboe@kernel.dk,
kbusch@kernel.org, hch@lst.de, sagi@grimberg.me,
emilne@redhat.com, hare@kernel.org
Cc: linux-block@vger.kernel.org, cgroups@vger.kernel.org,
linux-nvme@lists.infradead.org, linux-kernel@vger.kernel.org,
jmeneghi@redhat.com, jrani@purestorage.com,
randyj@purestorage.com
Subject: [PATCH v4 5/6] nvme: add 'latency' iopolicy
Date: Tue, 14 May 2024 13:53:21 -0400 [thread overview]
Message-ID: <20240514175322.19073-6-jmeneghi@redhat.com> (raw)
In-Reply-To: <20240514175322.19073-1-jmeneghi@redhat.com>
From: Hannes Reinecke <hare@kernel.org>
Add a latency-based I/O policy for multipathing. It uses the blk-nodelat
latency tracker to provide latencies for each node, and schedules
I/O on the path with the least latency for the submitting node.
Signed-off-by: Hannes Reinecke <hare@kernel.org>
[jmeneghi: fix CONFIG_BLK_NODE_LATENCY n and add latency iopolicy to modinfo]
Signed-off-by: John Meneghini <jmeneghi@redhat.com>
---
drivers/nvme/host/multipath.c | 62 ++++++++++++++++++++++++++++++-----
drivers/nvme/host/nvme.h | 1 +
2 files changed, 55 insertions(+), 8 deletions(-)
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 8702a40a1971..e9330bb1990b 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -18,6 +18,7 @@ static const char *nvme_iopolicy_names[] = {
[NVME_IOPOLICY_NUMA] = "numa",
[NVME_IOPOLICY_RR] = "round-robin",
[NVME_IOPOLICY_QD] = "queue-depth",
+ [NVME_IOPOLICY_LAT] = "latency",
};
static int iopolicy = NVME_IOPOLICY_NUMA;
@@ -32,6 +33,10 @@ static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
iopolicy = NVME_IOPOLICY_RR;
else if (!strncmp(val, "queue-depth", 11))
iopolicy = NVME_IOPOLICY_QD;
+#ifdef CONFIG_BLK_NODE_LATENCY
+ else if (!strncmp(val, "latency", 7))
+ iopolicy = NVME_IOPOLICY_LAT;
+#endif
else
return -EINVAL;
@@ -43,10 +48,36 @@ static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
}
+static int nvme_activate_iopolicy(struct nvme_subsystem *subsys, int iopolicy)
+{
+ struct nvme_ns_head *h;
+ struct nvme_ns *ns;
+ bool enable = iopolicy == NVME_IOPOLICY_LAT;
+ int ret = 0;
+
+ mutex_lock(&subsys->lock);
+ list_for_each_entry(h, &subsys->nsheads, entry) {
+ list_for_each_entry_rcu(ns, &h->list, siblings) {
+ if (enable) {
+ ret = blk_nlat_enable(ns->disk);
+ if (ret)
+ break;
+ } else
+ blk_nlat_disable(ns->disk);
+ }
+ }
+ mutex_unlock(&subsys->lock);
+ return ret;
+}
+
module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
&iopolicy, 0644);
MODULE_PARM_DESC(iopolicy,
+#if defined(CONFIG_BLK_NODE_LATENCY)
+ "Default multipath I/O policy; 'numa' (default) , 'round-robin', 'queue-depth' or 'latency'");
+#else
"Default multipath I/O policy; 'numa' (default) , 'round-robin' or 'queue-depth'");
+#endif
void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
{
@@ -250,13 +281,16 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
{
int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
struct nvme_ns *found = NULL, *fallback = NULL, *ns;
+ int iopolicy = READ_ONCE(head->subsys->iopolicy);
list_for_each_entry_rcu(ns, &head->list, siblings) {
if (nvme_path_is_disabled(ns))
continue;
- if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
+ if (iopolicy == NVME_IOPOLICY_NUMA)
distance = node_distance(node, ns->ctrl->numa_node);
+ else if (iopolicy == NVME_IOPOLICY_LAT)
+ distance = blk_nlat_latency(ns->disk, node);
else
distance = LOCAL_DISTANCE;
@@ -380,8 +414,8 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
{
- int iopolicy = READ_ONCE(head->subsys->iopolicy);
int node;
+ int iopolicy = READ_ONCE(head->subsys->iopolicy);
struct nvme_ns *ns;
/*
@@ -400,8 +434,8 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
if (iopolicy == NVME_IOPOLICY_RR)
return nvme_round_robin_path(head, node, ns);
-
- if (unlikely(!nvme_path_is_optimized(ns)))
+ if (iopolicy == NVME_IOPOLICY_LAT ||
+ unlikely(!nvme_path_is_optimized(ns)))
return __nvme_find_path(head, node);
return ns;
}
@@ -871,15 +905,18 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
{
struct nvme_subsystem *subsys =
container_of(dev, struct nvme_subsystem, dev);
- int i;
+ int i, ret;
for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
- nvme_subsys_iopolicy_update(subsys, i);
- return count;
+ ret = nvme_activate_iopolicy(subsys, i);
+ if (!ret) {
+ nvme_subsys_iopolicy_update(subsys, i);
+ return count;
+ }
+ return ret;
}
}
-
return -EINVAL;
}
SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
@@ -915,6 +952,15 @@ static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)
{
+ if (!blk_nlat_init(ns->disk) &&
+ READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_LAT) {
+ int ret = blk_nlat_enable(ns->disk);
+
+ if (unlikely(ret))
+ pr_warn("%s: Failed to enable latency tracking, error %d\n",
+ ns->disk->disk_name, ret);
+ }
+
if (nvme_ctrl_use_ana(ns->ctrl)) {
struct nvme_ana_group_desc desc = {
.grpid = anagrpid,
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 4e876524726a..56b78f21406a 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -406,6 +406,7 @@ enum nvme_iopolicy {
NVME_IOPOLICY_NUMA,
NVME_IOPOLICY_RR,
NVME_IOPOLICY_QD,
+ NVME_IOPOLICY_LAT,
};
struct nvme_subsystem {
--
2.39.3
next prev parent reply other threads:[~2024-05-14 17:53 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-05-14 17:53 [PATCH v4 0/6] block,nvme: queue-depth and latency I/O schedulers John Meneghini
2024-05-14 17:53 ` [PATCH v4 1/6] nvme: multipath: Implemented new iopolicy "queue-depth" John Meneghini
2024-05-20 14:46 ` Keith Busch
2024-05-14 17:53 ` [PATCH v4 2/6] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy John Meneghini
2024-05-14 17:53 ` [PATCH v4 3/6] nvme: multipath: Invalidate current_path when changing iopolicy John Meneghini
2024-05-14 17:53 ` [PATCH v4 4/6] block: track per-node I/O latency John Meneghini
2024-05-14 17:53 ` John Meneghini [this message]
2024-05-14 17:53 ` [PATCH v4 6/6] nvme: multipath: pr_notice when iopolicy changes John Meneghini
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240514175322.19073-6-jmeneghi@redhat.com \
--to=jmeneghi@redhat.com \
--cc=axboe@kernel.dk \
--cc=cgroups@vger.kernel.org \
--cc=emilne@redhat.com \
--cc=hare@kernel.org \
--cc=hch@lst.de \
--cc=josef@toxicpanda.com \
--cc=jrani@purestorage.com \
--cc=kbusch@kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=randyj@purestorage.com \
--cc=sagi@grimberg.me \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).