* [PATCH v2 1/4] ceph: use ktime_to_timespec64() rather than jiffies_to_timespec64()
2022-03-08 12:42 [PATCH v2 0/4] ceph: forward average read/write/metadata latency Venky Shankar
@ 2022-03-08 12:42 ` Venky Shankar
2022-03-08 12:42 ` [PATCH v2 2/4] ceph: track average r/w/m latency Venky Shankar
` (4 subsequent siblings)
5 siblings, 0 replies; 8+ messages in thread
From: Venky Shankar @ 2022-03-08 12:42 UTC (permalink / raw)
To: jlayton, idryomov; +Cc: xiubli, ceph-devel, Venky Shankar
Latencies are of type ktime_t, coverting from jiffies is incorrect.
Also, switch to "struct ceph_timespec" for r/w/m latencies.
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
fs/ceph/metric.c | 19 +++++++++----------
fs/ceph/metric.h | 11 ++++-------
2 files changed, 13 insertions(+), 17 deletions(-)
diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 0fcba68f9a99..454d2c93208e 100644
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -8,6 +8,12 @@
#include "metric.h"
#include "mds_client.h"
+static void ktime_to_ceph_timespec(struct ceph_timespec *ts, ktime_t val)
+{
+ struct timespec64 t = ktime_to_timespec64(val);
+ ceph_encode_timespec64(ts, &t);
+}
+
static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
struct ceph_mds_session *s)
{
@@ -26,7 +32,6 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
u64 nr_caps = atomic64_read(&m->total_caps);
u32 header_len = sizeof(struct ceph_metric_header);
struct ceph_msg *msg;
- struct timespec64 ts;
s64 sum;
s32 items = 0;
s32 len;
@@ -63,9 +68,7 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
read->header.compat = 1;
read->header.data_len = cpu_to_le32(sizeof(*read) - header_len);
sum = m->metric[METRIC_READ].latency_sum;
- jiffies_to_timespec64(sum, &ts);
- read->sec = cpu_to_le32(ts.tv_sec);
- read->nsec = cpu_to_le32(ts.tv_nsec);
+ ktime_to_ceph_timespec(&read->lat, sum);
items++;
/* encode the write latency metric */
@@ -75,9 +78,7 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
write->header.compat = 1;
write->header.data_len = cpu_to_le32(sizeof(*write) - header_len);
sum = m->metric[METRIC_WRITE].latency_sum;
- jiffies_to_timespec64(sum, &ts);
- write->sec = cpu_to_le32(ts.tv_sec);
- write->nsec = cpu_to_le32(ts.tv_nsec);
+ ktime_to_ceph_timespec(&write->lat, sum);
items++;
/* encode the metadata latency metric */
@@ -87,9 +88,7 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
meta->header.compat = 1;
meta->header.data_len = cpu_to_le32(sizeof(*meta) - header_len);
sum = m->metric[METRIC_METADATA].latency_sum;
- jiffies_to_timespec64(sum, &ts);
- meta->sec = cpu_to_le32(ts.tv_sec);
- meta->nsec = cpu_to_le32(ts.tv_nsec);
+ ktime_to_ceph_timespec(&meta->lat, sum);
items++;
/* encode the dentry lease metric */
diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index bb45608181e7..5b2bb2897056 100644
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -2,7 +2,7 @@
#ifndef _FS_CEPH_MDS_METRIC_H
#define _FS_CEPH_MDS_METRIC_H
-#include <linux/types.h>
+#include <linux/ceph/types.h>
#include <linux/percpu_counter.h>
#include <linux/ktime.h>
@@ -60,22 +60,19 @@ struct ceph_metric_cap {
/* metric read latency header */
struct ceph_metric_read_latency {
struct ceph_metric_header header;
- __le32 sec;
- __le32 nsec;
+ struct ceph_timespec lat;
} __packed;
/* metric write latency header */
struct ceph_metric_write_latency {
struct ceph_metric_header header;
- __le32 sec;
- __le32 nsec;
+ struct ceph_timespec lat;
} __packed;
/* metric metadata latency header */
struct ceph_metric_metadata_latency {
struct ceph_metric_header header;
- __le32 sec;
- __le32 nsec;
+ struct ceph_timespec lat;
} __packed;
/* metric dentry lease header */
--
2.31.1
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH v2 2/4] ceph: track average r/w/m latency
2022-03-08 12:42 [PATCH v2 0/4] ceph: forward average read/write/metadata latency Venky Shankar
2022-03-08 12:42 ` [PATCH v2 1/4] ceph: use ktime_to_timespec64() rather than jiffies_to_timespec64() Venky Shankar
@ 2022-03-08 12:42 ` Venky Shankar
2022-03-08 12:42 ` [PATCH v2 3/4] ceph: include average/stdev r/w/m latency in mds metrics Venky Shankar
` (3 subsequent siblings)
5 siblings, 0 replies; 8+ messages in thread
From: Venky Shankar @ 2022-03-08 12:42 UTC (permalink / raw)
To: jlayton, idryomov; +Cc: xiubli, ceph-devel, Venky Shankar
Make the math a bit simpler to understand (should not
effect execution speeds).
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
fs/ceph/metric.c | 29 +++++++++++++++--------------
fs/ceph/metric.h | 1 +
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 454d2c93208e..14b6af48b611 100644
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -249,6 +249,7 @@ int ceph_metric_init(struct ceph_client_metric *m)
metric->size_max = 0;
metric->total = 0;
metric->latency_sum = 0;
+ metric->latency_avg = 0;
metric->latency_sq_sum = 0;
metric->latency_min = KTIME_MAX;
metric->latency_max = 0;
@@ -306,20 +307,19 @@ void ceph_metric_destroy(struct ceph_client_metric *m)
max = new; \
}
-static inline void __update_stdev(ktime_t total, ktime_t lsum,
- ktime_t *sq_sump, ktime_t lat)
+static inline void __update_mean_and_stdev(ktime_t total, ktime_t *lavg,
+ ktime_t *sq_sump, ktime_t lat)
{
- ktime_t avg, sq;
-
- if (unlikely(total == 1))
- return;
-
- /* the sq is (lat - old_avg) * (lat - new_avg) */
- avg = DIV64_U64_ROUND_CLOSEST((lsum - lat), (total - 1));
- sq = lat - avg;
- avg = DIV64_U64_ROUND_CLOSEST(lsum, total);
- sq = sq * (lat - avg);
- *sq_sump += sq;
+ ktime_t avg;
+
+ if (unlikely(total == 1)) {
+ *lavg = lat;
+ } else {
+ /* the sq is (lat - old_avg) * (lat - new_avg) */
+ avg = *lavg + div64_s64(lat - *lavg, total);
+ *sq_sump += (lat - *lavg)*(lat - avg);
+ *lavg = avg;
+ }
}
void ceph_update_metrics(struct ceph_metric *m,
@@ -338,6 +338,7 @@ void ceph_update_metrics(struct ceph_metric *m,
METRIC_UPDATE_MIN_MAX(m->size_min, m->size_max, size);
m->latency_sum += lat;
METRIC_UPDATE_MIN_MAX(m->latency_min, m->latency_max, lat);
- __update_stdev(total, m->latency_sum, &m->latency_sq_sum, lat);
+ __update_mean_and_stdev(total, &m->latency_avg, &m->latency_sq_sum,
+ lat);
spin_unlock(&m->lock);
}
diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index 5b2bb2897056..c47ba0074e49 100644
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -137,6 +137,7 @@ struct ceph_metric {
u64 size_min;
u64 size_max;
ktime_t latency_sum;
+ ktime_t latency_avg;
ktime_t latency_sq_sum;
ktime_t latency_min;
ktime_t latency_max;
--
2.31.1
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH v2 3/4] ceph: include average/stdev r/w/m latency in mds metrics
2022-03-08 12:42 [PATCH v2 0/4] ceph: forward average read/write/metadata latency Venky Shankar
2022-03-08 12:42 ` [PATCH v2 1/4] ceph: use ktime_to_timespec64() rather than jiffies_to_timespec64() Venky Shankar
2022-03-08 12:42 ` [PATCH v2 2/4] ceph: track average r/w/m latency Venky Shankar
@ 2022-03-08 12:42 ` Venky Shankar
2022-03-08 12:42 ` [PATCH v2 4/4] ceph: use tracked average r/w/m latencies to display metrics in debugfs Venky Shankar
` (2 subsequent siblings)
5 siblings, 0 replies; 8+ messages in thread
From: Venky Shankar @ 2022-03-08 12:42 UTC (permalink / raw)
To: jlayton, idryomov; +Cc: xiubli, ceph-devel, Venky Shankar
stdev is computed in `cephfs-top` tool - clients forward
square of sums and IO count required to calculate stdev.
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
fs/ceph/metric.c | 15 +++++++++++---
fs/ceph/metric.h | 51 ++++++++++++++++++++++++++++++++++--------------
2 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 14b6af48b611..c47347d2e84e 100644
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -64,31 +64,40 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
/* encode the read latency metric */
read = (struct ceph_metric_read_latency *)(cap + 1);
read->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_READ_LATENCY);
- read->header.ver = 1;
+ read->header.ver = 2;
read->header.compat = 1;
read->header.data_len = cpu_to_le32(sizeof(*read) - header_len);
sum = m->metric[METRIC_READ].latency_sum;
ktime_to_ceph_timespec(&read->lat, sum);
+ ktime_to_ceph_timespec(&read->avg, m->metric[METRIC_READ].latency_avg);
+ read->sq_sum = cpu_to_le64(m->metric[METRIC_READ].latency_sq_sum);
+ read->count = cpu_to_le64(m->metric[METRIC_READ].total);
items++;
/* encode the write latency metric */
write = (struct ceph_metric_write_latency *)(read + 1);
write->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_WRITE_LATENCY);
- write->header.ver = 1;
+ write->header.ver = 2;
write->header.compat = 1;
write->header.data_len = cpu_to_le32(sizeof(*write) - header_len);
sum = m->metric[METRIC_WRITE].latency_sum;
ktime_to_ceph_timespec(&write->lat, sum);
+ ktime_to_ceph_timespec(&write->avg, m->metric[METRIC_WRITE].latency_avg);
+ write->sq_sum = cpu_to_le64(m->metric[METRIC_WRITE].latency_sq_sum);
+ write->count = cpu_to_le64(m->metric[METRIC_WRITE].total);
items++;
/* encode the metadata latency metric */
meta = (struct ceph_metric_metadata_latency *)(write + 1);
meta->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_METADATA_LATENCY);
- meta->header.ver = 1;
+ meta->header.ver = 2;
meta->header.compat = 1;
meta->header.data_len = cpu_to_le32(sizeof(*meta) - header_len);
sum = m->metric[METRIC_METADATA].latency_sum;
ktime_to_ceph_timespec(&meta->lat, sum);
+ ktime_to_ceph_timespec(&meta->avg, m->metric[METRIC_METADATA].latency_avg);
+ meta->sq_sum = cpu_to_le64(m->metric[METRIC_METADATA].latency_sq_sum);
+ meta->count = cpu_to_le64(m->metric[METRIC_METADATA].total);
items++;
/* encode the dentry lease metric */
diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index c47ba0074e49..0d0c44bd3332 100644
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -19,27 +19,39 @@ enum ceph_metric_type {
CLIENT_METRIC_TYPE_OPENED_INODES,
CLIENT_METRIC_TYPE_READ_IO_SIZES,
CLIENT_METRIC_TYPE_WRITE_IO_SIZES,
-
- CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_WRITE_IO_SIZES,
+ CLIENT_METRIC_TYPE_AVG_READ_LATENCY,
+ CLIENT_METRIC_TYPE_STDEV_READ_LATENCY,
+ CLIENT_METRIC_TYPE_AVG_WRITE_LATENCY,
+ CLIENT_METRIC_TYPE_STDEV_WRITE_LATENCY,
+ CLIENT_METRIC_TYPE_AVG_METADATA_LATENCY,
+ CLIENT_METRIC_TYPE_STDEV_METADATA_LATENCY,
+
+ CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_STDEV_METADATA_LATENCY,
};
/*
* This will always have the highest metric bit value
* as the last element of the array.
*/
-#define CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED { \
- CLIENT_METRIC_TYPE_CAP_INFO, \
- CLIENT_METRIC_TYPE_READ_LATENCY, \
- CLIENT_METRIC_TYPE_WRITE_LATENCY, \
- CLIENT_METRIC_TYPE_METADATA_LATENCY, \
- CLIENT_METRIC_TYPE_DENTRY_LEASE, \
- CLIENT_METRIC_TYPE_OPENED_FILES, \
- CLIENT_METRIC_TYPE_PINNED_ICAPS, \
- CLIENT_METRIC_TYPE_OPENED_INODES, \
- CLIENT_METRIC_TYPE_READ_IO_SIZES, \
- CLIENT_METRIC_TYPE_WRITE_IO_SIZES, \
- \
- CLIENT_METRIC_TYPE_MAX, \
+#define CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED { \
+ CLIENT_METRIC_TYPE_CAP_INFO, \
+ CLIENT_METRIC_TYPE_READ_LATENCY, \
+ CLIENT_METRIC_TYPE_WRITE_LATENCY, \
+ CLIENT_METRIC_TYPE_METADATA_LATENCY, \
+ CLIENT_METRIC_TYPE_DENTRY_LEASE, \
+ CLIENT_METRIC_TYPE_OPENED_FILES, \
+ CLIENT_METRIC_TYPE_PINNED_ICAPS, \
+ CLIENT_METRIC_TYPE_OPENED_INODES, \
+ CLIENT_METRIC_TYPE_READ_IO_SIZES, \
+ CLIENT_METRIC_TYPE_WRITE_IO_SIZES, \
+ CLIENT_METRIC_TYPE_AVG_READ_LATENCY, \
+ CLIENT_METRIC_TYPE_STDEV_READ_LATENCY, \
+ CLIENT_METRIC_TYPE_AVG_WRITE_LATENCY, \
+ CLIENT_METRIC_TYPE_STDEV_WRITE_LATENCY, \
+ CLIENT_METRIC_TYPE_AVG_METADATA_LATENCY, \
+ CLIENT_METRIC_TYPE_STDEV_METADATA_LATENCY, \
+ \
+ CLIENT_METRIC_TYPE_MAX, \
}
struct ceph_metric_header {
@@ -61,18 +73,27 @@ struct ceph_metric_cap {
struct ceph_metric_read_latency {
struct ceph_metric_header header;
struct ceph_timespec lat;
+ struct ceph_timespec avg;
+ __le64 sq_sum;
+ __le64 count;
} __packed;
/* metric write latency header */
struct ceph_metric_write_latency {
struct ceph_metric_header header;
struct ceph_timespec lat;
+ struct ceph_timespec avg;
+ __le64 sq_sum;
+ __le64 count;
} __packed;
/* metric metadata latency header */
struct ceph_metric_metadata_latency {
struct ceph_metric_header header;
struct ceph_timespec lat;
+ struct ceph_timespec avg;
+ __le64 sq_sum;
+ __le64 count;
} __packed;
/* metric dentry lease header */
--
2.31.1
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH v2 4/4] ceph: use tracked average r/w/m latencies to display metrics in debugfs
2022-03-08 12:42 [PATCH v2 0/4] ceph: forward average read/write/metadata latency Venky Shankar
` (2 preceding siblings ...)
2022-03-08 12:42 ` [PATCH v2 3/4] ceph: include average/stdev r/w/m latency in mds metrics Venky Shankar
@ 2022-03-08 12:42 ` Venky Shankar
2022-03-08 12:57 ` [PATCH v2 0/4] ceph: forward average read/write/metadata latency Xiubo Li
2022-03-08 15:58 ` Jeff Layton
5 siblings, 0 replies; 8+ messages in thread
From: Venky Shankar @ 2022-03-08 12:42 UTC (permalink / raw)
To: jlayton, idryomov; +Cc: xiubli, ceph-devel, Venky Shankar
[ jlayton: remove now-unused "sum" variable ]
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
fs/ceph/debugfs.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c
index 3cf7c9c1085b..bec3c4549c07 100644
--- a/fs/ceph/debugfs.c
+++ b/fs/ceph/debugfs.c
@@ -175,7 +175,7 @@ static int metrics_latency_show(struct seq_file *s, void *p)
struct ceph_fs_client *fsc = s->private;
struct ceph_client_metric *cm = &fsc->mdsc->metric;
struct ceph_metric *m;
- s64 total, sum, avg, min, max, sq;
+ s64 total, avg, min, max, sq;
int i;
seq_printf(s, "item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n");
@@ -185,8 +185,7 @@ static int metrics_latency_show(struct seq_file *s, void *p)
m = &cm->metric[i];
spin_lock(&m->lock);
total = m->total;
- sum = m->latency_sum;
- avg = total > 0 ? DIV64_U64_ROUND_CLOSEST(sum, total) : 0;
+ avg = m->latency_avg;
min = m->latency_min;
max = m->latency_max;
sq = m->latency_sq_sum;
--
2.31.1
^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH v2 0/4] ceph: forward average read/write/metadata latency
2022-03-08 12:42 [PATCH v2 0/4] ceph: forward average read/write/metadata latency Venky Shankar
` (3 preceding siblings ...)
2022-03-08 12:42 ` [PATCH v2 4/4] ceph: use tracked average r/w/m latencies to display metrics in debugfs Venky Shankar
@ 2022-03-08 12:57 ` Xiubo Li
2022-03-08 15:58 ` Jeff Layton
5 siblings, 0 replies; 8+ messages in thread
From: Xiubo Li @ 2022-03-08 12:57 UTC (permalink / raw)
To: Venky Shankar, jlayton, idryomov; +Cc: ceph-devel
On 3/8/22 8:42 PM, Venky Shankar wrote:
> v2
> - rename to_ceph_timespec() to ktime_to_ceph_timespec()
> - use ceph_encode_timespec64() helper
>
> Jeff,
>
> To apply these, please drop commit range f4bf256..840d9f0 from testing branch.
>
> Right now, cumulative read/write/metadata latencies are tracked
> and are periodically forwarded to the MDS. These meterics are not
> particularly useful. A much more useful metric is the average latency
> and standard deviation (stdev) which is what this series of patches
> aims to do.
>
> The userspace (libcephfs+tool) changes are here::
>
> https://github.com/ceph/ceph/pull/41397
>
> Note that the cumulative latencies are still forwarded to the MDS but
> the tool (cephfs-top) ignores it altogether.
>
> Latency standard deviation is calculated in `cephfs-top` tool.
>
> Venky Shankar (4):
> ceph: use ktime_to_timespec64() rather than jiffies_to_timespec64()
> ceph: track average r/w/m latency
> ceph: include average/stdev r/w/m latency in mds metrics
> ceph: use tracked average r/w/m latencies to display metrics in
> debugfs
>
> fs/ceph/debugfs.c | 5 ++--
> fs/ceph/metric.c | 63 +++++++++++++++++++++++++++--------------------
> fs/ceph/metric.h | 63 ++++++++++++++++++++++++++++++-----------------
> 3 files changed, 79 insertions(+), 52 deletions(-)
>
The series LGTM.
Reviewed-by: Xiubo Li <xiubli@redhat.com>
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH v2 0/4] ceph: forward average read/write/metadata latency
2022-03-08 12:42 [PATCH v2 0/4] ceph: forward average read/write/metadata latency Venky Shankar
` (4 preceding siblings ...)
2022-03-08 12:57 ` [PATCH v2 0/4] ceph: forward average read/write/metadata latency Xiubo Li
@ 2022-03-08 15:58 ` Jeff Layton
5 siblings, 0 replies; 8+ messages in thread
From: Jeff Layton @ 2022-03-08 15:58 UTC (permalink / raw)
To: Venky Shankar, idryomov; +Cc: xiubli, ceph-devel
On Tue, 2022-03-08 at 07:42 -0500, Venky Shankar wrote:
> v2
> - rename to_ceph_timespec() to ktime_to_ceph_timespec()
> - use ceph_encode_timespec64() helper
>
> Jeff,
>
> To apply these, please drop commit range f4bf256..840d9f0 from testing branch.
>
> Right now, cumulative read/write/metadata latencies are tracked
> and are periodically forwarded to the MDS. These meterics are not
> particularly useful. A much more useful metric is the average latency
> and standard deviation (stdev) which is what this series of patches
> aims to do.
>
> The userspace (libcephfs+tool) changes are here::
>
> https://github.com/ceph/ceph/pull/41397
>
> Note that the cumulative latencies are still forwarded to the MDS but
> the tool (cephfs-top) ignores it altogether.
>
> Latency standard deviation is calculated in `cephfs-top` tool.
>
> Venky Shankar (4):
> ceph: use ktime_to_timespec64() rather than jiffies_to_timespec64()
> ceph: track average r/w/m latency
> ceph: include average/stdev r/w/m latency in mds metrics
> ceph: use tracked average r/w/m latencies to display metrics in
> debugfs
>
> fs/ceph/debugfs.c | 5 ++--
> fs/ceph/metric.c | 63 +++++++++++++++++++++++++++--------------------
> fs/ceph/metric.h | 63 ++++++++++++++++++++++++++++++-----------------
> 3 files changed, 79 insertions(+), 52 deletions(-)
>
Thanks Venky. New version is now merged into testing.
--
Jeff Layton <jlayton@redhat.com>
^ permalink raw reply [flat|nested] 8+ messages in thread