* [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
@ 2020-11-26 3:47 xiubli
2021-03-22 2:30 ` Xiubo Li
2021-03-23 19:52 ` Jeff Layton
0 siblings, 2 replies; 7+ messages in thread
From: xiubli @ 2020-11-26 3:47 UTC (permalink / raw)
To: jlayton, idryomov; +Cc: zyan, pdonnell, ceph-devel, Xiubo Li
From: Xiubo Li <xiubli@redhat.com>
For the old ceph version, if it received this metric message containing
the send opened files/pinned caps/opened inodes metric info, it will
just ignore them.
URL: https://tracker.ceph.com/issues/46866
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 80 insertions(+), 2 deletions(-)
diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 5ec94bd4c1de..306bd599d940 100644
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
struct ceph_metric_write_latency *write;
struct ceph_metric_metadata_latency *meta;
struct ceph_metric_dlease *dlease;
+ struct ceph_opened_files *files;
+ struct ceph_pinned_icaps *icaps;
+ struct ceph_opened_inodes *inodes;
struct ceph_client_metric *m = &mdsc->metric;
u64 nr_caps = atomic64_read(&m->total_caps);
struct ceph_msg *msg;
@@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
s32 len;
len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
- + sizeof(*meta) + sizeof(*dlease);
+ + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
+ + sizeof(inodes);
msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
if (!msg) {
@@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
items++;
+ sum = percpu_counter_sum(&m->total_inodes);
+
+ /* encode the opened files metric */
+ files = (struct ceph_opened_files *)(dlease + 1);
+ files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
+ files->ver = 1;
+ files->compat = 1;
+ files->data_len = cpu_to_le32(sizeof(*files) - 10);
+ files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
+ files->total = cpu_to_le64(sum);
+ items++;
+
+ /* encode the pinned icaps metric */
+ icaps = (struct ceph_pinned_icaps *)(files + 1);
+ icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
+ icaps->ver = 1;
+ icaps->compat = 1;
+ icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
+ icaps->pinned_icaps = cpu_to_le64(nr_caps);
+ icaps->total = cpu_to_le64(sum);
+ items++;
+
+ /* encode the opened inodes metric */
+ inodes = (struct ceph_opened_inodes *)(icaps + 1);
+ inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
+ inodes->ver = 1;
+ inodes->compat = 1;
+ inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
+ inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
+ inodes->total = cpu_to_le64(sum);
+ items++;
+
put_unaligned_le32(items, &head->num);
msg->front.iov_len = len;
msg->hdr.version = cpu_to_le16(1);
diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index af6038ff39d4..4ceb462135d7 100644
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -14,8 +14,11 @@ enum ceph_metric_type {
CLIENT_METRIC_TYPE_WRITE_LATENCY,
CLIENT_METRIC_TYPE_METADATA_LATENCY,
CLIENT_METRIC_TYPE_DENTRY_LEASE,
+ CLIENT_METRIC_TYPE_OPENED_FILES,
+ CLIENT_METRIC_TYPE_PINNED_ICAPS,
+ CLIENT_METRIC_TYPE_OPENED_INODES,
- CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
+ CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
};
/*
@@ -28,6 +31,9 @@ enum ceph_metric_type {
CLIENT_METRIC_TYPE_WRITE_LATENCY, \
CLIENT_METRIC_TYPE_METADATA_LATENCY, \
CLIENT_METRIC_TYPE_DENTRY_LEASE, \
+ CLIENT_METRIC_TYPE_OPENED_FILES, \
+ CLIENT_METRIC_TYPE_PINNED_ICAPS, \
+ CLIENT_METRIC_TYPE_OPENED_INODES, \
\
CLIENT_METRIC_TYPE_MAX, \
}
@@ -94,6 +100,42 @@ struct ceph_metric_dlease {
__le64 total;
} __packed;
+/* metric opened files header */
+struct ceph_opened_files {
+ __le32 type; /* ceph metric type */
+
+ __u8 ver;
+ __u8 compat;
+
+ __le32 data_len; /* length of sizeof(opened_files + total) */
+ __le64 opened_files;
+ __le64 total;
+} __packed;
+
+/* metric pinned i_caps header */
+struct ceph_pinned_icaps {
+ __le32 type; /* ceph metric type */
+
+ __u8 ver;
+ __u8 compat;
+
+ __le32 data_len; /* length of sizeof(pinned_icaps + total) */
+ __le64 pinned_icaps;
+ __le64 total;
+} __packed;
+
+/* metric opened inodes header */
+struct ceph_opened_inodes {
+ __le32 type; /* ceph metric type */
+
+ __u8 ver;
+ __u8 compat;
+
+ __le32 data_len; /* length of sizeof(opened_inodes + total) */
+ __le64 opened_inodes;
+ __le64 total;
+} __packed;
+
struct ceph_metric_head {
__le32 num; /* the number of metrics that will be sent */
} __packed;
--
2.27.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
2020-11-26 3:47 [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon xiubli
@ 2021-03-22 2:30 ` Xiubo Li
2021-03-22 11:55 ` Jeff Layton
2021-03-23 19:52 ` Jeff Layton
1 sibling, 1 reply; 7+ messages in thread
From: Xiubo Li @ 2021-03-22 2:30 UTC (permalink / raw)
To: jlayton; +Cc: pdonnell, ceph-devel, Ilya Dryomov
Hi Jeff,
Ping.
The MDS side patch[1] have been merged.
[1] https://github.com/ceph/ceph/pull/37945
Thanks
On 2020/11/26 11:47, xiubli@redhat.com wrote:
> From: Xiubo Li <xiubli@redhat.com>
>
> For the old ceph version, if it received this metric message containing
> the send opened files/pinned caps/opened inodes metric info, it will
> just ignore them.
>
> URL: https://tracker.ceph.com/issues/46866
> Signed-off-by: Xiubo Li <xiubli@redhat.com>
> ---
> fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
> fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
> 2 files changed, 80 insertions(+), 2 deletions(-)
>
> diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> index 5ec94bd4c1de..306bd599d940 100644
> --- a/fs/ceph/metric.c
> +++ b/fs/ceph/metric.c
> @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> struct ceph_metric_write_latency *write;
> struct ceph_metric_metadata_latency *meta;
> struct ceph_metric_dlease *dlease;
> + struct ceph_opened_files *files;
> + struct ceph_pinned_icaps *icaps;
> + struct ceph_opened_inodes *inodes;
> struct ceph_client_metric *m = &mdsc->metric;
> u64 nr_caps = atomic64_read(&m->total_caps);
> struct ceph_msg *msg;
> @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> s32 len;
>
> len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
> - + sizeof(*meta) + sizeof(*dlease);
> + + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
> + + sizeof(inodes);
>
> msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
> if (!msg) {
> @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
> items++;
>
> + sum = percpu_counter_sum(&m->total_inodes);
> +
> + /* encode the opened files metric */
> + files = (struct ceph_opened_files *)(dlease + 1);
> + files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
> + files->ver = 1;
> + files->compat = 1;
> + files->data_len = cpu_to_le32(sizeof(*files) - 10);
> + files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
> + files->total = cpu_to_le64(sum);
> + items++;
> +
> + /* encode the pinned icaps metric */
> + icaps = (struct ceph_pinned_icaps *)(files + 1);
> + icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
> + icaps->ver = 1;
> + icaps->compat = 1;
> + icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
> + icaps->pinned_icaps = cpu_to_le64(nr_caps);
> + icaps->total = cpu_to_le64(sum);
> + items++;
> +
> + /* encode the opened inodes metric */
> + inodes = (struct ceph_opened_inodes *)(icaps + 1);
> + inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
> + inodes->ver = 1;
> + inodes->compat = 1;
> + inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
> + inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
> + inodes->total = cpu_to_le64(sum);
> + items++;
> +
> put_unaligned_le32(items, &head->num);
> msg->front.iov_len = len;
> msg->hdr.version = cpu_to_le16(1);
> diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
> index af6038ff39d4..4ceb462135d7 100644
> --- a/fs/ceph/metric.h
> +++ b/fs/ceph/metric.h
> @@ -14,8 +14,11 @@ enum ceph_metric_type {
> CLIENT_METRIC_TYPE_WRITE_LATENCY,
> CLIENT_METRIC_TYPE_METADATA_LATENCY,
> CLIENT_METRIC_TYPE_DENTRY_LEASE,
> + CLIENT_METRIC_TYPE_OPENED_FILES,
> + CLIENT_METRIC_TYPE_PINNED_ICAPS,
> + CLIENT_METRIC_TYPE_OPENED_INODES,
>
> - CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
> + CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
> };
>
> /*
> @@ -28,6 +31,9 @@ enum ceph_metric_type {
> CLIENT_METRIC_TYPE_WRITE_LATENCY, \
> CLIENT_METRIC_TYPE_METADATA_LATENCY, \
> CLIENT_METRIC_TYPE_DENTRY_LEASE, \
> + CLIENT_METRIC_TYPE_OPENED_FILES, \
> + CLIENT_METRIC_TYPE_PINNED_ICAPS, \
> + CLIENT_METRIC_TYPE_OPENED_INODES, \
> \
> CLIENT_METRIC_TYPE_MAX, \
> }
> @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
> __le64 total;
> } __packed;
>
> +/* metric opened files header */
> +struct ceph_opened_files {
> + __le32 type; /* ceph metric type */
> +
> + __u8 ver;
> + __u8 compat;
> +
> + __le32 data_len; /* length of sizeof(opened_files + total) */
> + __le64 opened_files;
> + __le64 total;
> +} __packed;
> +
> +/* metric pinned i_caps header */
> +struct ceph_pinned_icaps {
> + __le32 type; /* ceph metric type */
> +
> + __u8 ver;
> + __u8 compat;
> +
> + __le32 data_len; /* length of sizeof(pinned_icaps + total) */
> + __le64 pinned_icaps;
> + __le64 total;
> +} __packed;
> +
> +/* metric opened inodes header */
> +struct ceph_opened_inodes {
> + __le32 type; /* ceph metric type */
> +
> + __u8 ver;
> + __u8 compat;
> +
> + __le32 data_len; /* length of sizeof(opened_inodes + total) */
> + __le64 opened_inodes;
> + __le64 total;
> +} __packed;
> +
> struct ceph_metric_head {
> __le32 num; /* the number of metrics that will be sent */
> } __packed;
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
2021-03-22 2:30 ` Xiubo Li
@ 2021-03-22 11:55 ` Jeff Layton
2021-03-22 12:32 ` Xiubo Li
2021-03-24 4:54 ` Xiubo Li
0 siblings, 2 replies; 7+ messages in thread
From: Jeff Layton @ 2021-03-22 11:55 UTC (permalink / raw)
To: Xiubo Li; +Cc: pdonnell, ceph-devel, Ilya Dryomov
Oh! I hadn't realized that this patch was still unmerged.
Merged into testing branch. I'd have pulled this into testing a while
ago if I had realized. Let me know if I miss any in the future (or am
missing any now).
Cheers,
Jeff
On Mon, 2021-03-22 at 10:30 +0800, Xiubo Li wrote:
> Hi Jeff,
>
> Ping.
>
> The MDS side patch[1] have been merged.
>
> [1] https://github.com/ceph/ceph/pull/37945
>
> Thanks
>
>
> On 2020/11/26 11:47, xiubli@redhat.com wrote:
> > From: Xiubo Li <xiubli@redhat.com>
> >
> > For the old ceph version, if it received this metric message containing
> > the send opened files/pinned caps/opened inodes metric info, it will
> > just ignore them.
> >
> > URL: https://tracker.ceph.com/issues/46866
> > Signed-off-by: Xiubo Li <xiubli@redhat.com>
> > ---
> > fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
> > fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
> > 2 files changed, 80 insertions(+), 2 deletions(-)
> >
> > diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> > index 5ec94bd4c1de..306bd599d940 100644
> > --- a/fs/ceph/metric.c
> > +++ b/fs/ceph/metric.c
> > @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> > struct ceph_metric_write_latency *write;
> > struct ceph_metric_metadata_latency *meta;
> > struct ceph_metric_dlease *dlease;
> > + struct ceph_opened_files *files;
> > + struct ceph_pinned_icaps *icaps;
> > + struct ceph_opened_inodes *inodes;
> > struct ceph_client_metric *m = &mdsc->metric;
> > u64 nr_caps = atomic64_read(&m->total_caps);
> > struct ceph_msg *msg;
> > @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> > s32 len;
> >
> >
> > len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
> > - + sizeof(*meta) + sizeof(*dlease);
> > + + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
> > + + sizeof(inodes);
> >
> >
> > msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
> > if (!msg) {
> > @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> > dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
> > items++;
> >
> >
> > + sum = percpu_counter_sum(&m->total_inodes);
> > +
> > + /* encode the opened files metric */
> > + files = (struct ceph_opened_files *)(dlease + 1);
> > + files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
> > + files->ver = 1;
> > + files->compat = 1;
> > + files->data_len = cpu_to_le32(sizeof(*files) - 10);
> > + files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
> > + files->total = cpu_to_le64(sum);
> > + items++;
> > +
> > + /* encode the pinned icaps metric */
> > + icaps = (struct ceph_pinned_icaps *)(files + 1);
> > + icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
> > + icaps->ver = 1;
> > + icaps->compat = 1;
> > + icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
> > + icaps->pinned_icaps = cpu_to_le64(nr_caps);
> > + icaps->total = cpu_to_le64(sum);
> > + items++;
> > +
> > + /* encode the opened inodes metric */
> > + inodes = (struct ceph_opened_inodes *)(icaps + 1);
> > + inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
> > + inodes->ver = 1;
> > + inodes->compat = 1;
> > + inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
> > + inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
> > + inodes->total = cpu_to_le64(sum);
> > + items++;
> > +
> > put_unaligned_le32(items, &head->num);
> > msg->front.iov_len = len;
> > msg->hdr.version = cpu_to_le16(1);
> > diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
> > index af6038ff39d4..4ceb462135d7 100644
> > --- a/fs/ceph/metric.h
> > +++ b/fs/ceph/metric.h
> > @@ -14,8 +14,11 @@ enum ceph_metric_type {
> > CLIENT_METRIC_TYPE_WRITE_LATENCY,
> > CLIENT_METRIC_TYPE_METADATA_LATENCY,
> > CLIENT_METRIC_TYPE_DENTRY_LEASE,
> > + CLIENT_METRIC_TYPE_OPENED_FILES,
> > + CLIENT_METRIC_TYPE_PINNED_ICAPS,
> > + CLIENT_METRIC_TYPE_OPENED_INODES,
> >
> >
> > - CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
> > + CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
> > };
> >
> >
> > /*
> > @@ -28,6 +31,9 @@ enum ceph_metric_type {
> > CLIENT_METRIC_TYPE_WRITE_LATENCY, \
> > CLIENT_METRIC_TYPE_METADATA_LATENCY, \
> > CLIENT_METRIC_TYPE_DENTRY_LEASE, \
> > + CLIENT_METRIC_TYPE_OPENED_FILES, \
> > + CLIENT_METRIC_TYPE_PINNED_ICAPS, \
> > + CLIENT_METRIC_TYPE_OPENED_INODES, \
> > \
> > CLIENT_METRIC_TYPE_MAX, \
> > }
> > @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
> > __le64 total;
> > } __packed;
> >
> >
> > +/* metric opened files header */
> > +struct ceph_opened_files {
> > + __le32 type; /* ceph metric type */
> > +
> > + __u8 ver;
> > + __u8 compat;
> > +
> > + __le32 data_len; /* length of sizeof(opened_files + total) */
> > + __le64 opened_files;
> > + __le64 total;
> > +} __packed;
> > +
> > +/* metric pinned i_caps header */
> > +struct ceph_pinned_icaps {
> > + __le32 type; /* ceph metric type */
> > +
> > + __u8 ver;
> > + __u8 compat;
> > +
> > + __le32 data_len; /* length of sizeof(pinned_icaps + total) */
> > + __le64 pinned_icaps;
> > + __le64 total;
> > +} __packed;
> > +
> > +/* metric opened inodes header */
> > +struct ceph_opened_inodes {
> > + __le32 type; /* ceph metric type */
> > +
> > + __u8 ver;
> > + __u8 compat;
> > +
> > + __le32 data_len; /* length of sizeof(opened_inodes + total) */
> > + __le64 opened_inodes;
> > + __le64 total;
> > +} __packed;
> > +
> > struct ceph_metric_head {
> > __le32 num; /* the number of metrics that will be sent */
> > } __packed;
>
>
--
Jeff Layton <jlayton@kernel.org>
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
2021-03-22 11:55 ` Jeff Layton
@ 2021-03-22 12:32 ` Xiubo Li
2021-03-23 19:51 ` Jeff Layton
2021-03-24 4:54 ` Xiubo Li
1 sibling, 1 reply; 7+ messages in thread
From: Xiubo Li @ 2021-03-22 12:32 UTC (permalink / raw)
To: Jeff Layton; +Cc: pdonnell, ceph-devel, Ilya Dryomov
On 2021/3/22 19:55, Jeff Layton wrote:
> Oh! I hadn't realized that this patch was still unmerged.
No worry, this patch was waiting the MDS side patch, which was just
merged last week :-)
I almost forgot this patch too before I saw the ceph tracker today.
> Merged into testing branch. I'd have pulled this into testing a while
> ago if I had realized. Let me know if I miss any in the future (or am
> missing any now).
Sure, np.
BRs
Xiubo
> Cheers,
> Jeff
>
> On Mon, 2021-03-22 at 10:30 +0800, Xiubo Li wrote:
>> Hi Jeff,
>>
>> Ping.
>>
>> The MDS side patch[1] have been merged.
>>
>> [1] https://github.com/ceph/ceph/pull/37945
>>
>> Thanks
>>
>>
>> On 2020/11/26 11:47, xiubli@redhat.com wrote:
>>> From: Xiubo Li <xiubli@redhat.com>
>>>
>>> For the old ceph version, if it received this metric message containing
>>> the send opened files/pinned caps/opened inodes metric info, it will
>>> just ignore them.
>>>
>>> URL: https://tracker.ceph.com/issues/46866
>>> Signed-off-by: Xiubo Li <xiubli@redhat.com>
>>> ---
>>> fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
>>> fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
>>> 2 files changed, 80 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
>>> index 5ec94bd4c1de..306bd599d940 100644
>>> --- a/fs/ceph/metric.c
>>> +++ b/fs/ceph/metric.c
>>> @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>>> struct ceph_metric_write_latency *write;
>>> struct ceph_metric_metadata_latency *meta;
>>> struct ceph_metric_dlease *dlease;
>>> + struct ceph_opened_files *files;
>>> + struct ceph_pinned_icaps *icaps;
>>> + struct ceph_opened_inodes *inodes;
>>> struct ceph_client_metric *m = &mdsc->metric;
>>> u64 nr_caps = atomic64_read(&m->total_caps);
>>> struct ceph_msg *msg;
>>> @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>>> s32 len;
>>>
>>>
>>> len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
>>> - + sizeof(*meta) + sizeof(*dlease);
>>> + + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
>>> + + sizeof(inodes);
>>>
>>>
>>> msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
>>> if (!msg) {
>>> @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>>> dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
>>> items++;
>>>
>>>
>>> + sum = percpu_counter_sum(&m->total_inodes);
>>> +
>>> + /* encode the opened files metric */
>>> + files = (struct ceph_opened_files *)(dlease + 1);
>>> + files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
>>> + files->ver = 1;
>>> + files->compat = 1;
>>> + files->data_len = cpu_to_le32(sizeof(*files) - 10);
>>> + files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
>>> + files->total = cpu_to_le64(sum);
>>> + items++;
>>> +
>>> + /* encode the pinned icaps metric */
>>> + icaps = (struct ceph_pinned_icaps *)(files + 1);
>>> + icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
>>> + icaps->ver = 1;
>>> + icaps->compat = 1;
>>> + icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
>>> + icaps->pinned_icaps = cpu_to_le64(nr_caps);
>>> + icaps->total = cpu_to_le64(sum);
>>> + items++;
>>> +
>>> + /* encode the opened inodes metric */
>>> + inodes = (struct ceph_opened_inodes *)(icaps + 1);
>>> + inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
>>> + inodes->ver = 1;
>>> + inodes->compat = 1;
>>> + inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
>>> + inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
>>> + inodes->total = cpu_to_le64(sum);
>>> + items++;
>>> +
>>> put_unaligned_le32(items, &head->num);
>>> msg->front.iov_len = len;
>>> msg->hdr.version = cpu_to_le16(1);
>>> diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
>>> index af6038ff39d4..4ceb462135d7 100644
>>> --- a/fs/ceph/metric.h
>>> +++ b/fs/ceph/metric.h
>>> @@ -14,8 +14,11 @@ enum ceph_metric_type {
>>> CLIENT_METRIC_TYPE_WRITE_LATENCY,
>>> CLIENT_METRIC_TYPE_METADATA_LATENCY,
>>> CLIENT_METRIC_TYPE_DENTRY_LEASE,
>>> + CLIENT_METRIC_TYPE_OPENED_FILES,
>>> + CLIENT_METRIC_TYPE_PINNED_ICAPS,
>>> + CLIENT_METRIC_TYPE_OPENED_INODES,
>>>
>>>
>>> - CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
>>> + CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
>>> };
>>>
>>>
>>> /*
>>> @@ -28,6 +31,9 @@ enum ceph_metric_type {
>>> CLIENT_METRIC_TYPE_WRITE_LATENCY, \
>>> CLIENT_METRIC_TYPE_METADATA_LATENCY, \
>>> CLIENT_METRIC_TYPE_DENTRY_LEASE, \
>>> + CLIENT_METRIC_TYPE_OPENED_FILES, \
>>> + CLIENT_METRIC_TYPE_PINNED_ICAPS, \
>>> + CLIENT_METRIC_TYPE_OPENED_INODES, \
>>> \
>>> CLIENT_METRIC_TYPE_MAX, \
>>> }
>>> @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
>>> __le64 total;
>>> } __packed;
>>>
>>>
>>> +/* metric opened files header */
>>> +struct ceph_opened_files {
>>> + __le32 type; /* ceph metric type */
>>> +
>>> + __u8 ver;
>>> + __u8 compat;
>>> +
>>> + __le32 data_len; /* length of sizeof(opened_files + total) */
>>> + __le64 opened_files;
>>> + __le64 total;
>>> +} __packed;
>>> +
>>> +/* metric pinned i_caps header */
>>> +struct ceph_pinned_icaps {
>>> + __le32 type; /* ceph metric type */
>>> +
>>> + __u8 ver;
>>> + __u8 compat;
>>> +
>>> + __le32 data_len; /* length of sizeof(pinned_icaps + total) */
>>> + __le64 pinned_icaps;
>>> + __le64 total;
>>> +} __packed;
>>> +
>>> +/* metric opened inodes header */
>>> +struct ceph_opened_inodes {
>>> + __le32 type; /* ceph metric type */
>>> +
>>> + __u8 ver;
>>> + __u8 compat;
>>> +
>>> + __le32 data_len; /* length of sizeof(opened_inodes + total) */
>>> + __le64 opened_inodes;
>>> + __le64 total;
>>> +} __packed;
>>> +
>>> struct ceph_metric_head {
>>> __le32 num; /* the number of metrics that will be sent */
>>> } __packed;
>>
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
2021-03-22 12:32 ` Xiubo Li
@ 2021-03-23 19:51 ` Jeff Layton
0 siblings, 0 replies; 7+ messages in thread
From: Jeff Layton @ 2021-03-23 19:51 UTC (permalink / raw)
To: Xiubo Li; +Cc: pdonnell, ceph-devel, Ilya Dryomov
On Mon, 2021-03-22 at 20:32 +0800, Xiubo Li wrote:
> On 2021/3/22 19:55, Jeff Layton wrote:
> > Oh! I hadn't realized that this patch was still unmerged.
>
> No worry, this patch was waiting the MDS side patch, which was just
> merged last week :-)
>
> I almost forgot this patch too before I saw the ceph tracker today.
>
> > Merged into testing branch. I'd have pulled this into testing a while
> > ago if I had realized. Let me know if I miss any in the future (or am
> > missing any now).
>
> Sure, np.
>
> BRs
>
> Xiubo
>
After doing some testing with this patch, I hit the following KASAN pop
today. I've backed this patch out of the testing branch for now:
[ 110.969263] ==================================================================
[ 110.972933] BUG: KASAN: slab-out-of-bounds in metric_delayed_work+0x5e9/0x880 [ceph]
[ 110.976096] Write of size 8 at addr ffff88811b76ab90 by task kworker/7:2/613
[ 110.978972]
[ 110.979668] CPU: 7 PID: 613 Comm: kworker/7:2 Tainted: G E 5.12.0-rc2+ #70
[ 110.983015] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.14.0-1.fc33 04/01/2014
[ 110.986467] Workqueue: events metric_delayed_work [ceph]
[ 110.988803] Call Trace:
[ 110.989869] dump_stack+0xa5/0xdc
[ 110.991297] print_address_description.constprop.0+0x18/0x160
[ 110.993659] ? metric_delayed_work+0x5e9/0x880 [ceph]
[ 110.995892] ? metric_delayed_work+0x5e9/0x880 [ceph]
[ 110.998108] kasan_report.cold+0x7f/0x111
[ 110.999818] ? _raw_spin_lock_irq+0x61/0x90
[ 111.001563] ? metric_delayed_work+0x5e9/0x880 [ceph]
[ 111.003814] kasan_check_range+0xf9/0x1e0
[ 111.005503] metric_delayed_work+0x5e9/0x880 [ceph]
[ 111.007688] ? ceph_caps_for_mode+0x40/0x40 [ceph]
[ 111.009855] process_one_work+0x525/0x9b0
[ 111.011597] ? pwq_dec_nr_in_flight+0x110/0x110
[ 111.013504] ? lock_acquired+0x2fe/0x560
[ 111.015230] worker_thread+0x2ea/0x6e0
[ 111.016840] ? __kthread_parkme+0xc0/0xe0
[ 111.018534] ? process_one_work+0x9b0/0x9b0
[ 111.020262] kthread+0x1fb/0x220
[ 111.021622] ? __kthread_bind_mask+0x70/0x70
[ 111.023423] ret_from_fork+0x22/0x30
[ 111.025004]
[ 111.025687] Allocated by task 613:
[ 111.027105] kasan_save_stack+0x1b/0x40
[ 111.028740] __kasan_kmalloc+0x7a/0x90
[ 111.029583] ceph_kvmalloc+0xc7/0x110 [libceph]
[ 111.030611] ceph_msg_new2+0x15d/0x270 [libceph]
[ 111.031537] metric_delayed_work+0x1c5/0x880 [ceph]
[ 111.032414] process_one_work+0x525/0x9b0
[ 111.033093] worker_thread+0x2ea/0x6e0
[ 111.033763] kthread+0x1fb/0x220
[ 111.034329] ret_from_fork+0x22/0x30
[ 111.038203]
[ 111.041730] The buggy address belongs to the object at ffff88811b76ab00
[ 111.041730] which belongs to the cache kmalloc-192 of size 192
[ 111.050535] The buggy address is located 144 bytes inside of
[ 111.050535] 192-byte region [ffff88811b76ab00, ffff88811b76abc0)
[ 111.059194] The buggy address belongs to the page:
[ 111.063348] page:0000000052971d41 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x11b76a
[ 111.068337] head:0000000052971d41 order:1 compound_mapcount:0
[ 111.073364] flags: 0x17ffffc0010200(slab|head)
[ 111.078020] raw: 0017ffffc0010200 0000000000000000 0000000100000001 ffff888100042a00
[ 111.083177] raw: 0000000000000000 0000000080200020 00000001ffffffff 0000000000000000
[ 111.088350] page dumped because: kasan: bad access detected
[ 111.093064]
[ 111.096773] Memory state around the buggy address:
[ 111.101168] ffff88811b76aa80: 00 00 fc fc fc fc fc fc fc fc fc fc fc fc fc fc
[ 111.106154] ffff88811b76ab00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
[ 111.111073] >ffff88811b76ab80: 00 00 06 fc fc fc fc fc fc fc fc fc fc fc fc fc
[ 111.115973] ^
[ 111.120219] ffff88811b76ac00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
[ 111.125263] ffff88811b76ac80: 00 fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
[ 111.130227] ==================================================================
--
Jeff Layton <jlayton@kernel.org>
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
2021-03-22 11:55 ` Jeff Layton
2021-03-22 12:32 ` Xiubo Li
@ 2021-03-24 4:54 ` Xiubo Li
1 sibling, 0 replies; 7+ messages in thread
From: Xiubo Li @ 2021-03-24 4:54 UTC (permalink / raw)
To: Jeff Layton; +Cc: pdonnell, ceph-devel, Ilya Dryomov
On 2021/3/24 12:53, Xiubo Li wrote:
[...]
> > diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> > index 5ec94bd4c1de..306bd599d940 100644
> > --- a/fs/ceph/metric.c
> > +++ b/fs/ceph/metric.c
> > @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct
> ceph_mds_client *mdsc,
> > struct ceph_metric_write_latency *write;
> > struct ceph_metric_metadata_latency *meta;
> > struct ceph_metric_dlease *dlease;
> > + struct ceph_opened_files *files;
> > + struct ceph_pinned_icaps *icaps;
> > + struct ceph_opened_inodes *inodes;
> > struct ceph_client_metric *m = &mdsc->metric;
> > u64 nr_caps = atomic64_read(&m->total_caps);
> > struct ceph_msg *msg;
> > @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct
> ceph_mds_client *mdsc,
> > s32 len;
> >
> > len = sizeof(*head) + sizeof(*cap) + sizeof(*read) +
> sizeof(*write)
> > - + sizeof(*meta) + sizeof(*dlease);
> > + + sizeof(*meta) + sizeof(*dlease) + sizeof(files) +
> sizeof(icaps)
> > + + sizeof(inodes);
> >
>
> These sizeof values look wrong. The buffer requires more than pointers
> for those. You probably want:
>
> ... + sizeof(*files) + sizeof(*icaps) + sizeof(*inodes);
>
Yeah, will fix it.
Thanks
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
2020-11-26 3:47 [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon xiubli
2021-03-22 2:30 ` Xiubo Li
@ 2021-03-23 19:52 ` Jeff Layton
1 sibling, 0 replies; 7+ messages in thread
From: Jeff Layton @ 2021-03-23 19:52 UTC (permalink / raw)
To: xiubli, idryomov; +Cc: zyan, pdonnell, ceph-devel
On Thu, 2020-11-26 at 11:47 +0800, xiubli@redhat.com wrote:
> From: Xiubo Li <xiubli@redhat.com>
>
> For the old ceph version, if it received this metric message containing
> the send opened files/pinned caps/opened inodes metric info, it will
> just ignore them.
>
> URL: https://tracker.ceph.com/issues/46866
> Signed-off-by: Xiubo Li <xiubli@redhat.com>
> ---
> fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
> fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
> 2 files changed, 80 insertions(+), 2 deletions(-)
>
> diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> index 5ec94bd4c1de..306bd599d940 100644
> --- a/fs/ceph/metric.c
> +++ b/fs/ceph/metric.c
> @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> struct ceph_metric_write_latency *write;
> struct ceph_metric_metadata_latency *meta;
> struct ceph_metric_dlease *dlease;
> + struct ceph_opened_files *files;
> + struct ceph_pinned_icaps *icaps;
> + struct ceph_opened_inodes *inodes;
> struct ceph_client_metric *m = &mdsc->metric;
> u64 nr_caps = atomic64_read(&m->total_caps);
> struct ceph_msg *msg;
> @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> s32 len;
>
> len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
> - + sizeof(*meta) + sizeof(*dlease);
> + + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
> + + sizeof(inodes);
>
These sizeof values look wrong. The buffer requires more than pointers
for those. You probably want:
... + sizeof(*files) + sizeof(*icaps) + sizeof(*inodes);
> msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
> if (!msg) {
> @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
> items++;
>
> + sum = percpu_counter_sum(&m->total_inodes);
> +
> + /* encode the opened files metric */
> + files = (struct ceph_opened_files *)(dlease + 1);
> + files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
> + files->ver = 1;
> + files->compat = 1;
> + files->data_len = cpu_to_le32(sizeof(*files) - 10);
> + files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
> + files->total = cpu_to_le64(sum);
> + items++;
> +
> + /* encode the pinned icaps metric */
> + icaps = (struct ceph_pinned_icaps *)(files + 1);
> + icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
> + icaps->ver = 1;
> + icaps->compat = 1;
> + icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
> + icaps->pinned_icaps = cpu_to_le64(nr_caps);
> + icaps->total = cpu_to_le64(sum);
> + items++;
> +
> + /* encode the opened inodes metric */
> + inodes = (struct ceph_opened_inodes *)(icaps + 1);
> + inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
> + inodes->ver = 1;
> + inodes->compat = 1;
> + inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
> + inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
> + inodes->total = cpu_to_le64(sum);
> + items++;
> +
> put_unaligned_le32(items, &head->num);
> msg->front.iov_len = len;
> msg->hdr.version = cpu_to_le16(1);
> diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
> index af6038ff39d4..4ceb462135d7 100644
> --- a/fs/ceph/metric.h
> +++ b/fs/ceph/metric.h
> @@ -14,8 +14,11 @@ enum ceph_metric_type {
> CLIENT_METRIC_TYPE_WRITE_LATENCY,
> CLIENT_METRIC_TYPE_METADATA_LATENCY,
> CLIENT_METRIC_TYPE_DENTRY_LEASE,
> + CLIENT_METRIC_TYPE_OPENED_FILES,
> + CLIENT_METRIC_TYPE_PINNED_ICAPS,
> + CLIENT_METRIC_TYPE_OPENED_INODES,
>
> - CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
> + CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
> };
>
> /*
> @@ -28,6 +31,9 @@ enum ceph_metric_type {
> CLIENT_METRIC_TYPE_WRITE_LATENCY, \
> CLIENT_METRIC_TYPE_METADATA_LATENCY, \
> CLIENT_METRIC_TYPE_DENTRY_LEASE, \
> + CLIENT_METRIC_TYPE_OPENED_FILES, \
> + CLIENT_METRIC_TYPE_PINNED_ICAPS, \
> + CLIENT_METRIC_TYPE_OPENED_INODES, \
> \
> CLIENT_METRIC_TYPE_MAX, \
> }
> @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
> __le64 total;
> } __packed;
>
> +/* metric opened files header */
> +struct ceph_opened_files {
> + __le32 type; /* ceph metric type */
> +
> + __u8 ver;
> + __u8 compat;
> +
> + __le32 data_len; /* length of sizeof(opened_files + total) */
> + __le64 opened_files;
> + __le64 total;
> +} __packed;
> +
> +/* metric pinned i_caps header */
> +struct ceph_pinned_icaps {
> + __le32 type; /* ceph metric type */
> +
> + __u8 ver;
> + __u8 compat;
> +
> + __le32 data_len; /* length of sizeof(pinned_icaps + total) */
> + __le64 pinned_icaps;
> + __le64 total;
> +} __packed;
> +
> +/* metric opened inodes header */
> +struct ceph_opened_inodes {
> + __le32 type; /* ceph metric type */
> +
> + __u8 ver;
> + __u8 compat;
> +
> + __le32 data_len; /* length of sizeof(opened_inodes + total) */
> + __le64 opened_inodes;
> + __le64 total;
> +} __packed;
> +
> struct ceph_metric_head {
> __le32 num; /* the number of metrics that will be sent */
> } __packed;
--
Jeff Layton <jlayton@kernel.org>
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2021-03-24 4:56 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-11-26 3:47 [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon xiubli
2021-03-22 2:30 ` Xiubo Li
2021-03-22 11:55 ` Jeff Layton
2021-03-22 12:32 ` Xiubo Li
2021-03-23 19:51 ` Jeff Layton
2021-03-24 4:54 ` Xiubo Li
2021-03-23 19:52 ` Jeff Layton
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).