From: Venky Shankar <vshankar@redhat.com>
To: jlayton@redhat.com, pdonnell@redhat.com, xiubli@redhat.com
Cc: ceph-devel@vger.kernel.org, Venky Shankar <vshankar@redhat.com>
Subject: [PATCH v1 2/4] ceph: track average/stdev r/w/m latency
Date: Mon, 13 Sep 2021 18:43:09 +0530 [thread overview]
Message-ID: <20210913131311.1347903-3-vshankar@redhat.com> (raw)
In-Reply-To: <20210913131311.1347903-1-vshankar@redhat.com>
The math involved in tracking average and standard deviation
for r/w/m latencies looks incorrect. Fix that up. Also, change
the variable name that tracks standard deviation (*_sq_sum) to
*_stdev.
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
fs/ceph/debugfs.c | 6 +++---
fs/ceph/metric.c | 45 ++++++++++++++++++++++++---------------------
fs/ceph/metric.h | 9 ++++++---
3 files changed, 33 insertions(+), 27 deletions(-)
diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c
index f9ff70704423..9153bc233e08 100644
--- a/fs/ceph/debugfs.c
+++ b/fs/ceph/debugfs.c
@@ -176,7 +176,7 @@ static int metric_show(struct seq_file *s, void *p)
avg = total > 0 ? DIV64_U64_ROUND_CLOSEST(sum, total) : 0;
min = m->read_latency_min;
max = m->read_latency_max;
- sq = m->read_latency_sq_sum;
+ sq = m->read_latency_stdev;
spin_unlock(&m->read_latency_lock);
CEPH_METRIC_SHOW("read", total, avg, min, max, sq);
@@ -186,7 +186,7 @@ static int metric_show(struct seq_file *s, void *p)
avg = total > 0 ? DIV64_U64_ROUND_CLOSEST(sum, total) : 0;
min = m->write_latency_min;
max = m->write_latency_max;
- sq = m->write_latency_sq_sum;
+ sq = m->write_latency_stdev;
spin_unlock(&m->write_latency_lock);
CEPH_METRIC_SHOW("write", total, avg, min, max, sq);
@@ -196,7 +196,7 @@ static int metric_show(struct seq_file *s, void *p)
avg = total > 0 ? DIV64_U64_ROUND_CLOSEST(sum, total) : 0;
min = m->metadata_latency_min;
max = m->metadata_latency_max;
- sq = m->metadata_latency_sq_sum;
+ sq = m->metadata_latency_stdev;
spin_unlock(&m->metadata_latency_lock);
CEPH_METRIC_SHOW("metadata", total, avg, min, max, sq);
diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 19bca20cf7e1..e00b1c2ce25b 100644
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -184,21 +184,24 @@ int ceph_metric_init(struct ceph_client_metric *m)
goto err_i_caps_mis;
spin_lock_init(&m->read_latency_lock);
- m->read_latency_sq_sum = 0;
+ m->read_latency_stdev = 0;
+ m->avg_read_latency = 0;
m->read_latency_min = KTIME_MAX;
m->read_latency_max = 0;
m->total_reads = 0;
m->read_latency_sum = 0;
spin_lock_init(&m->write_latency_lock);
- m->write_latency_sq_sum = 0;
+ m->write_latency_stdev = 0;
+ m->avg_write_latency = 0;
m->write_latency_min = KTIME_MAX;
m->write_latency_max = 0;
m->total_writes = 0;
m->write_latency_sum = 0;
spin_lock_init(&m->metadata_latency_lock);
- m->metadata_latency_sq_sum = 0;
+ m->metadata_latency_stdev = 0;
+ m->avg_metadata_latency = 0;
m->metadata_latency_min = KTIME_MAX;
m->metadata_latency_max = 0;
m->total_metadatas = 0;
@@ -250,10 +253,10 @@ void ceph_metric_destroy(struct ceph_client_metric *m)
}
static inline void __update_latency(ktime_t *totalp, ktime_t *lsump,
- ktime_t *min, ktime_t *max,
- ktime_t *sq_sump, ktime_t lat)
+ ktime_t *lavgp, ktime_t *min, ktime_t *max,
+ ktime_t *lstdev, ktime_t lat)
{
- ktime_t total, avg, sq, lsum;
+ ktime_t total, avg, stdev, lsum;
total = ++(*totalp);
lsum = (*lsump += lat);
@@ -263,15 +266,15 @@ static inline void __update_latency(ktime_t *totalp, ktime_t *lsump,
if (unlikely(lat > *max))
*max = lat;
- if (unlikely(total == 1))
- return;
-
- /* the sq is (lat - old_avg) * (lat - new_avg) */
- avg = DIV64_U64_ROUND_CLOSEST((lsum - lat), (total - 1));
- sq = lat - avg;
- avg = DIV64_U64_ROUND_CLOSEST(lsum, total);
- sq = sq * (lat - avg);
- *sq_sump += sq;
+ if (unlikely(total == 1)) {
+ *lavgp = lat;
+ *lstdev = 0;
+ } else {
+ avg = *lavgp + div64_s64(lat - *lavgp, total);
+ stdev = *lstdev + (lat - *lavgp)*(lat - avg);
+ *lstdev = int_sqrt(div64_u64(stdev, total - 1));
+ *lavgp = avg;
+ }
}
void ceph_update_read_latency(struct ceph_client_metric *m,
@@ -285,8 +288,8 @@ void ceph_update_read_latency(struct ceph_client_metric *m,
spin_lock(&m->read_latency_lock);
__update_latency(&m->total_reads, &m->read_latency_sum,
- &m->read_latency_min, &m->read_latency_max,
- &m->read_latency_sq_sum, lat);
+ &m->avg_read_latency, &m->read_latency_min,
+ &m->read_latency_max, &m->read_latency_stdev, lat);
spin_unlock(&m->read_latency_lock);
}
@@ -301,8 +304,8 @@ void ceph_update_write_latency(struct ceph_client_metric *m,
spin_lock(&m->write_latency_lock);
__update_latency(&m->total_writes, &m->write_latency_sum,
- &m->write_latency_min, &m->write_latency_max,
- &m->write_latency_sq_sum, lat);
+ &m->avg_write_latency, &m->write_latency_min,
+ &m->write_latency_max, &m->write_latency_stdev, lat);
spin_unlock(&m->write_latency_lock);
}
@@ -317,7 +320,7 @@ void ceph_update_metadata_latency(struct ceph_client_metric *m,
spin_lock(&m->metadata_latency_lock);
__update_latency(&m->total_metadatas, &m->metadata_latency_sum,
- &m->metadata_latency_min, &m->metadata_latency_max,
- &m->metadata_latency_sq_sum, lat);
+ &m->avg_metadata_latency, &m->metadata_latency_min,
+ &m->metadata_latency_max, &m->metadata_latency_stdev, lat);
spin_unlock(&m->metadata_latency_lock);
}
diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index 1151d64dbcbc..136689fa0aec 100644
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -108,21 +108,24 @@ struct ceph_client_metric {
spinlock_t read_latency_lock;
u64 total_reads;
ktime_t read_latency_sum;
- ktime_t read_latency_sq_sum;
+ ktime_t avg_read_latency;
+ ktime_t read_latency_stdev;
ktime_t read_latency_min;
ktime_t read_latency_max;
spinlock_t write_latency_lock;
u64 total_writes;
ktime_t write_latency_sum;
- ktime_t write_latency_sq_sum;
+ ktime_t avg_write_latency;
+ ktime_t write_latency_stdev;
ktime_t write_latency_min;
ktime_t write_latency_max;
spinlock_t metadata_latency_lock;
u64 total_metadatas;
ktime_t metadata_latency_sum;
- ktime_t metadata_latency_sq_sum;
+ ktime_t avg_metadata_latency;
+ ktime_t metadata_latency_stdev;
ktime_t metadata_latency_min;
ktime_t metadata_latency_max;
--
2.27.0
next prev parent reply other threads:[~2021-09-13 13:13 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-09-13 13:13 [PATCH v1 0/4] ceph: forward average read/write/metadata latency Venky Shankar
2021-09-13 13:13 ` [PATCH v1 1/4] ceph: use "struct ceph_timespec" for r/w/m latencies Venky Shankar
2021-09-13 13:13 ` Venky Shankar [this message]
2021-09-13 13:13 ` [PATCH v1 3/4] ceph: include average/stddev r/w/m latency in mds metrics Venky Shankar
2021-09-13 13:13 ` [PATCH v1 4/4] ceph: use tracked average r/w/m latencies to display metrics in debugfs Venky Shankar
2021-09-13 15:13 ` [PATCH v1 0/4] ceph: forward average read/write/metadata latency Jeff Layton
2021-09-13 15:21 ` Jeff Layton
2021-09-13 15:41 ` Venky Shankar
2021-09-14 12:53 ` Xiubo Li
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210913131311.1347903-3-vshankar@redhat.com \
--to=vshankar@redhat.com \
--cc=ceph-devel@vger.kernel.org \
--cc=jlayton@redhat.com \
--cc=pdonnell@redhat.com \
--cc=xiubli@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).