ceph-devel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
* [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
@ 2020-11-26  3:47 xiubli
  2021-03-22  2:30 ` Xiubo Li
  2021-03-23 19:52 ` Jeff Layton
  0 siblings, 2 replies; 7+ messages in thread
From: xiubli @ 2020-11-26  3:47 UTC (permalink / raw)
  To: jlayton, idryomov; +Cc: zyan, pdonnell, ceph-devel, Xiubo Li

From: Xiubo Li <xiubli@redhat.com>

For the old ceph version, if it received this metric message containing
the send opened files/pinned caps/opened inodes metric info, it will
just ignore them.

URL: https://tracker.ceph.com/issues/46866
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
 fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 5ec94bd4c1de..306bd599d940 100644
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
 	struct ceph_metric_write_latency *write;
 	struct ceph_metric_metadata_latency *meta;
 	struct ceph_metric_dlease *dlease;
+	struct ceph_opened_files *files;
+	struct ceph_pinned_icaps *icaps;
+	struct ceph_opened_inodes *inodes;
 	struct ceph_client_metric *m = &mdsc->metric;
 	u64 nr_caps = atomic64_read(&m->total_caps);
 	struct ceph_msg *msg;
@@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
 	s32 len;
 
 	len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
-	      + sizeof(*meta) + sizeof(*dlease);
+	      + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
+	      + sizeof(inodes);
 
 	msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
 	if (!msg) {
@@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
 	dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
 	items++;
 
+	sum = percpu_counter_sum(&m->total_inodes);
+
+	/* encode the opened files metric */
+	files = (struct ceph_opened_files *)(dlease + 1);
+	files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
+	files->ver = 1;
+	files->compat = 1;
+	files->data_len = cpu_to_le32(sizeof(*files) - 10);
+	files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
+	files->total = cpu_to_le64(sum);
+	items++;
+
+	/* encode the pinned icaps metric */
+	icaps = (struct ceph_pinned_icaps *)(files + 1);
+	icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
+	icaps->ver = 1;
+	icaps->compat = 1;
+	icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
+	icaps->pinned_icaps = cpu_to_le64(nr_caps);
+	icaps->total = cpu_to_le64(sum);
+	items++;
+
+	/* encode the opened inodes metric */
+	inodes = (struct ceph_opened_inodes *)(icaps + 1);
+	inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
+	inodes->ver = 1;
+	inodes->compat = 1;
+	inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
+	inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
+	inodes->total = cpu_to_le64(sum);
+	items++;
+
 	put_unaligned_le32(items, &head->num);
 	msg->front.iov_len = len;
 	msg->hdr.version = cpu_to_le16(1);
diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index af6038ff39d4..4ceb462135d7 100644
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -14,8 +14,11 @@ enum ceph_metric_type {
 	CLIENT_METRIC_TYPE_WRITE_LATENCY,
 	CLIENT_METRIC_TYPE_METADATA_LATENCY,
 	CLIENT_METRIC_TYPE_DENTRY_LEASE,
+	CLIENT_METRIC_TYPE_OPENED_FILES,
+	CLIENT_METRIC_TYPE_PINNED_ICAPS,
+	CLIENT_METRIC_TYPE_OPENED_INODES,
 
-	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
+	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
 };
 
 /*
@@ -28,6 +31,9 @@ enum ceph_metric_type {
 	CLIENT_METRIC_TYPE_WRITE_LATENCY,	\
 	CLIENT_METRIC_TYPE_METADATA_LATENCY,	\
 	CLIENT_METRIC_TYPE_DENTRY_LEASE,	\
+	CLIENT_METRIC_TYPE_OPENED_FILES,	\
+	CLIENT_METRIC_TYPE_PINNED_ICAPS,	\
+	CLIENT_METRIC_TYPE_OPENED_INODES,	\
 						\
 	CLIENT_METRIC_TYPE_MAX,			\
 }
@@ -94,6 +100,42 @@ struct ceph_metric_dlease {
 	__le64 total;
 } __packed;
 
+/* metric opened files header */
+struct ceph_opened_files {
+	__le32 type;     /* ceph metric type */
+
+	__u8  ver;
+	__u8  compat;
+
+	__le32 data_len; /* length of sizeof(opened_files + total) */
+	__le64 opened_files;
+	__le64 total;
+} __packed;
+
+/* metric pinned i_caps header */
+struct ceph_pinned_icaps {
+	__le32 type;     /* ceph metric type */
+
+	__u8  ver;
+	__u8  compat;
+
+	__le32 data_len; /* length of sizeof(pinned_icaps + total) */
+	__le64 pinned_icaps;
+	__le64 total;
+} __packed;
+
+/* metric opened inodes header */
+struct ceph_opened_inodes {
+	__le32 type;     /* ceph metric type */
+
+	__u8  ver;
+	__u8  compat;
+
+	__le32 data_len; /* length of sizeof(opened_inodes + total) */
+	__le64 opened_inodes;
+	__le64 total;
+} __packed;
+
 struct ceph_metric_head {
 	__le32 num;	/* the number of metrics that will be sent */
 } __packed;
-- 
2.27.0


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
  2020-11-26  3:47 [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon xiubli
@ 2021-03-22  2:30 ` Xiubo Li
  2021-03-22 11:55   ` Jeff Layton
  2021-03-23 19:52 ` Jeff Layton
  1 sibling, 1 reply; 7+ messages in thread
From: Xiubo Li @ 2021-03-22  2:30 UTC (permalink / raw)
  To: jlayton; +Cc: pdonnell, ceph-devel, Ilya Dryomov

Hi Jeff,

Ping.

The MDS side patch[1] have been merged.

[1] https://github.com/ceph/ceph/pull/37945

Thanks


On 2020/11/26 11:47, xiubli@redhat.com wrote:
> From: Xiubo Li <xiubli@redhat.com>
>
> For the old ceph version, if it received this metric message containing
> the send opened files/pinned caps/opened inodes metric info, it will
> just ignore them.
>
> URL: https://tracker.ceph.com/issues/46866
> Signed-off-by: Xiubo Li <xiubli@redhat.com>
> ---
>   fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
>   fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
>   2 files changed, 80 insertions(+), 2 deletions(-)
>
> diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> index 5ec94bd4c1de..306bd599d940 100644
> --- a/fs/ceph/metric.c
> +++ b/fs/ceph/metric.c
> @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>   	struct ceph_metric_write_latency *write;
>   	struct ceph_metric_metadata_latency *meta;
>   	struct ceph_metric_dlease *dlease;
> +	struct ceph_opened_files *files;
> +	struct ceph_pinned_icaps *icaps;
> +	struct ceph_opened_inodes *inodes;
>   	struct ceph_client_metric *m = &mdsc->metric;
>   	u64 nr_caps = atomic64_read(&m->total_caps);
>   	struct ceph_msg *msg;
> @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>   	s32 len;
>   
>   	len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
> -	      + sizeof(*meta) + sizeof(*dlease);
> +	      + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
> +	      + sizeof(inodes);
>   
>   	msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
>   	if (!msg) {
> @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>   	dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
>   	items++;
>   
> +	sum = percpu_counter_sum(&m->total_inodes);
> +
> +	/* encode the opened files metric */
> +	files = (struct ceph_opened_files *)(dlease + 1);
> +	files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
> +	files->ver = 1;
> +	files->compat = 1;
> +	files->data_len = cpu_to_le32(sizeof(*files) - 10);
> +	files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
> +	files->total = cpu_to_le64(sum);
> +	items++;
> +
> +	/* encode the pinned icaps metric */
> +	icaps = (struct ceph_pinned_icaps *)(files + 1);
> +	icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
> +	icaps->ver = 1;
> +	icaps->compat = 1;
> +	icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
> +	icaps->pinned_icaps = cpu_to_le64(nr_caps);
> +	icaps->total = cpu_to_le64(sum);
> +	items++;
> +
> +	/* encode the opened inodes metric */
> +	inodes = (struct ceph_opened_inodes *)(icaps + 1);
> +	inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
> +	inodes->ver = 1;
> +	inodes->compat = 1;
> +	inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
> +	inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
> +	inodes->total = cpu_to_le64(sum);
> +	items++;
> +
>   	put_unaligned_le32(items, &head->num);
>   	msg->front.iov_len = len;
>   	msg->hdr.version = cpu_to_le16(1);
> diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
> index af6038ff39d4..4ceb462135d7 100644
> --- a/fs/ceph/metric.h
> +++ b/fs/ceph/metric.h
> @@ -14,8 +14,11 @@ enum ceph_metric_type {
>   	CLIENT_METRIC_TYPE_WRITE_LATENCY,
>   	CLIENT_METRIC_TYPE_METADATA_LATENCY,
>   	CLIENT_METRIC_TYPE_DENTRY_LEASE,
> +	CLIENT_METRIC_TYPE_OPENED_FILES,
> +	CLIENT_METRIC_TYPE_PINNED_ICAPS,
> +	CLIENT_METRIC_TYPE_OPENED_INODES,
>   
> -	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
> +	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
>   };
>   
>   /*
> @@ -28,6 +31,9 @@ enum ceph_metric_type {
>   	CLIENT_METRIC_TYPE_WRITE_LATENCY,	\
>   	CLIENT_METRIC_TYPE_METADATA_LATENCY,	\
>   	CLIENT_METRIC_TYPE_DENTRY_LEASE,	\
> +	CLIENT_METRIC_TYPE_OPENED_FILES,	\
> +	CLIENT_METRIC_TYPE_PINNED_ICAPS,	\
> +	CLIENT_METRIC_TYPE_OPENED_INODES,	\
>   						\
>   	CLIENT_METRIC_TYPE_MAX,			\
>   }
> @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
>   	__le64 total;
>   } __packed;
>   
> +/* metric opened files header */
> +struct ceph_opened_files {
> +	__le32 type;     /* ceph metric type */
> +
> +	__u8  ver;
> +	__u8  compat;
> +
> +	__le32 data_len; /* length of sizeof(opened_files + total) */
> +	__le64 opened_files;
> +	__le64 total;
> +} __packed;
> +
> +/* metric pinned i_caps header */
> +struct ceph_pinned_icaps {
> +	__le32 type;     /* ceph metric type */
> +
> +	__u8  ver;
> +	__u8  compat;
> +
> +	__le32 data_len; /* length of sizeof(pinned_icaps + total) */
> +	__le64 pinned_icaps;
> +	__le64 total;
> +} __packed;
> +
> +/* metric opened inodes header */
> +struct ceph_opened_inodes {
> +	__le32 type;     /* ceph metric type */
> +
> +	__u8  ver;
> +	__u8  compat;
> +
> +	__le32 data_len; /* length of sizeof(opened_inodes + total) */
> +	__le64 opened_inodes;
> +	__le64 total;
> +} __packed;
> +
>   struct ceph_metric_head {
>   	__le32 num;	/* the number of metrics that will be sent */
>   } __packed;



^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
  2021-03-22  2:30 ` Xiubo Li
@ 2021-03-22 11:55   ` Jeff Layton
  2021-03-22 12:32     ` Xiubo Li
  2021-03-24  4:54     ` Xiubo Li
  0 siblings, 2 replies; 7+ messages in thread
From: Jeff Layton @ 2021-03-22 11:55 UTC (permalink / raw)
  To: Xiubo Li; +Cc: pdonnell, ceph-devel, Ilya Dryomov

Oh! I hadn't realized that this patch was still unmerged.

Merged into testing branch. I'd have pulled this into testing a while
ago if I had realized. Let me know if I miss any in the future (or am
missing any now).

Cheers,
Jeff

On Mon, 2021-03-22 at 10:30 +0800, Xiubo Li wrote:
> Hi Jeff,
> 
> Ping.
> 
> The MDS side patch[1] have been merged.
> 
> [1] https://github.com/ceph/ceph/pull/37945
> 
> Thanks
> 
> 
> On 2020/11/26 11:47, xiubli@redhat.com wrote:
> > From: Xiubo Li <xiubli@redhat.com>
> > 
> > For the old ceph version, if it received this metric message containing
> > the send opened files/pinned caps/opened inodes metric info, it will
> > just ignore them.
> > 
> > URL: https://tracker.ceph.com/issues/46866
> > Signed-off-by: Xiubo Li <xiubli@redhat.com>
> > ---
> >   fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
> >   fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
> >   2 files changed, 80 insertions(+), 2 deletions(-)
> > 
> > diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> > index 5ec94bd4c1de..306bd599d940 100644
> > --- a/fs/ceph/metric.c
> > +++ b/fs/ceph/metric.c
> > @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> >   	struct ceph_metric_write_latency *write;
> >   	struct ceph_metric_metadata_latency *meta;
> >   	struct ceph_metric_dlease *dlease;
> > +	struct ceph_opened_files *files;
> > +	struct ceph_pinned_icaps *icaps;
> > +	struct ceph_opened_inodes *inodes;
> >   	struct ceph_client_metric *m = &mdsc->metric;
> >   	u64 nr_caps = atomic64_read(&m->total_caps);
> >   	struct ceph_msg *msg;
> > @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> >   	s32 len;
> >   
> > 
> >   	len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
> > -	      + sizeof(*meta) + sizeof(*dlease);
> > +	      + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
> > +	      + sizeof(inodes);
> >   
> > 
> >   	msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
> >   	if (!msg) {
> > @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
> >   	dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
> >   	items++;
> >   
> > 
> > +	sum = percpu_counter_sum(&m->total_inodes);
> > +
> > +	/* encode the opened files metric */
> > +	files = (struct ceph_opened_files *)(dlease + 1);
> > +	files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
> > +	files->ver = 1;
> > +	files->compat = 1;
> > +	files->data_len = cpu_to_le32(sizeof(*files) - 10);
> > +	files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
> > +	files->total = cpu_to_le64(sum);
> > +	items++;
> > +
> > +	/* encode the pinned icaps metric */
> > +	icaps = (struct ceph_pinned_icaps *)(files + 1);
> > +	icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
> > +	icaps->ver = 1;
> > +	icaps->compat = 1;
> > +	icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
> > +	icaps->pinned_icaps = cpu_to_le64(nr_caps);
> > +	icaps->total = cpu_to_le64(sum);
> > +	items++;
> > +
> > +	/* encode the opened inodes metric */
> > +	inodes = (struct ceph_opened_inodes *)(icaps + 1);
> > +	inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
> > +	inodes->ver = 1;
> > +	inodes->compat = 1;
> > +	inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
> > +	inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
> > +	inodes->total = cpu_to_le64(sum);
> > +	items++;
> > +
> >   	put_unaligned_le32(items, &head->num);
> >   	msg->front.iov_len = len;
> >   	msg->hdr.version = cpu_to_le16(1);
> > diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
> > index af6038ff39d4..4ceb462135d7 100644
> > --- a/fs/ceph/metric.h
> > +++ b/fs/ceph/metric.h
> > @@ -14,8 +14,11 @@ enum ceph_metric_type {
> >   	CLIENT_METRIC_TYPE_WRITE_LATENCY,
> >   	CLIENT_METRIC_TYPE_METADATA_LATENCY,
> >   	CLIENT_METRIC_TYPE_DENTRY_LEASE,
> > +	CLIENT_METRIC_TYPE_OPENED_FILES,
> > +	CLIENT_METRIC_TYPE_PINNED_ICAPS,
> > +	CLIENT_METRIC_TYPE_OPENED_INODES,
> >   
> > 
> > -	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
> > +	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
> >   };
> >   
> > 
> >   /*
> > @@ -28,6 +31,9 @@ enum ceph_metric_type {
> >   	CLIENT_METRIC_TYPE_WRITE_LATENCY,	\
> >   	CLIENT_METRIC_TYPE_METADATA_LATENCY,	\
> >   	CLIENT_METRIC_TYPE_DENTRY_LEASE,	\
> > +	CLIENT_METRIC_TYPE_OPENED_FILES,	\
> > +	CLIENT_METRIC_TYPE_PINNED_ICAPS,	\
> > +	CLIENT_METRIC_TYPE_OPENED_INODES,	\
> >   						\
> >   	CLIENT_METRIC_TYPE_MAX,			\
> >   }
> > @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
> >   	__le64 total;
> >   } __packed;
> >   
> > 
> > +/* metric opened files header */
> > +struct ceph_opened_files {
> > +	__le32 type;     /* ceph metric type */
> > +
> > +	__u8  ver;
> > +	__u8  compat;
> > +
> > +	__le32 data_len; /* length of sizeof(opened_files + total) */
> > +	__le64 opened_files;
> > +	__le64 total;
> > +} __packed;
> > +
> > +/* metric pinned i_caps header */
> > +struct ceph_pinned_icaps {
> > +	__le32 type;     /* ceph metric type */
> > +
> > +	__u8  ver;
> > +	__u8  compat;
> > +
> > +	__le32 data_len; /* length of sizeof(pinned_icaps + total) */
> > +	__le64 pinned_icaps;
> > +	__le64 total;
> > +} __packed;
> > +
> > +/* metric opened inodes header */
> > +struct ceph_opened_inodes {
> > +	__le32 type;     /* ceph metric type */
> > +
> > +	__u8  ver;
> > +	__u8  compat;
> > +
> > +	__le32 data_len; /* length of sizeof(opened_inodes + total) */
> > +	__le64 opened_inodes;
> > +	__le64 total;
> > +} __packed;
> > +
> >   struct ceph_metric_head {
> >   	__le32 num;	/* the number of metrics that will be sent */
> >   } __packed;
> 
> 

-- 
Jeff Layton <jlayton@kernel.org>


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
  2021-03-22 11:55   ` Jeff Layton
@ 2021-03-22 12:32     ` Xiubo Li
  2021-03-23 19:51       ` Jeff Layton
  2021-03-24  4:54     ` Xiubo Li
  1 sibling, 1 reply; 7+ messages in thread
From: Xiubo Li @ 2021-03-22 12:32 UTC (permalink / raw)
  To: Jeff Layton; +Cc: pdonnell, ceph-devel, Ilya Dryomov

On 2021/3/22 19:55, Jeff Layton wrote:
> Oh! I hadn't realized that this patch was still unmerged.

No worry, this patch was waiting the MDS side patch, which was just 
merged last week :-)

I almost forgot this patch too before I saw the ceph tracker today.

> Merged into testing branch. I'd have pulled this into testing a while
> ago if I had realized. Let me know if I miss any in the future (or am
> missing any now).

Sure, np.

BRs

Xiubo



> Cheers,
> Jeff
>
> On Mon, 2021-03-22 at 10:30 +0800, Xiubo Li wrote:
>> Hi Jeff,
>>
>> Ping.
>>
>> The MDS side patch[1] have been merged.
>>
>> [1] https://github.com/ceph/ceph/pull/37945
>>
>> Thanks
>>
>>
>> On 2020/11/26 11:47, xiubli@redhat.com wrote:
>>> From: Xiubo Li <xiubli@redhat.com>
>>>
>>> For the old ceph version, if it received this metric message containing
>>> the send opened files/pinned caps/opened inodes metric info, it will
>>> just ignore them.
>>>
>>> URL: https://tracker.ceph.com/issues/46866
>>> Signed-off-by: Xiubo Li <xiubli@redhat.com>
>>> ---
>>>    fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
>>>    fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
>>>    2 files changed, 80 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
>>> index 5ec94bd4c1de..306bd599d940 100644
>>> --- a/fs/ceph/metric.c
>>> +++ b/fs/ceph/metric.c
>>> @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>>>    	struct ceph_metric_write_latency *write;
>>>    	struct ceph_metric_metadata_latency *meta;
>>>    	struct ceph_metric_dlease *dlease;
>>> +	struct ceph_opened_files *files;
>>> +	struct ceph_pinned_icaps *icaps;
>>> +	struct ceph_opened_inodes *inodes;
>>>    	struct ceph_client_metric *m = &mdsc->metric;
>>>    	u64 nr_caps = atomic64_read(&m->total_caps);
>>>    	struct ceph_msg *msg;
>>> @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>>>    	s32 len;
>>>    
>>>
>>>    	len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
>>> -	      + sizeof(*meta) + sizeof(*dlease);
>>> +	      + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
>>> +	      + sizeof(inodes);
>>>    
>>>
>>>    	msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
>>>    	if (!msg) {
>>> @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>>>    	dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
>>>    	items++;
>>>    
>>>
>>> +	sum = percpu_counter_sum(&m->total_inodes);
>>> +
>>> +	/* encode the opened files metric */
>>> +	files = (struct ceph_opened_files *)(dlease + 1);
>>> +	files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
>>> +	files->ver = 1;
>>> +	files->compat = 1;
>>> +	files->data_len = cpu_to_le32(sizeof(*files) - 10);
>>> +	files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
>>> +	files->total = cpu_to_le64(sum);
>>> +	items++;
>>> +
>>> +	/* encode the pinned icaps metric */
>>> +	icaps = (struct ceph_pinned_icaps *)(files + 1);
>>> +	icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
>>> +	icaps->ver = 1;
>>> +	icaps->compat = 1;
>>> +	icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
>>> +	icaps->pinned_icaps = cpu_to_le64(nr_caps);
>>> +	icaps->total = cpu_to_le64(sum);
>>> +	items++;
>>> +
>>> +	/* encode the opened inodes metric */
>>> +	inodes = (struct ceph_opened_inodes *)(icaps + 1);
>>> +	inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
>>> +	inodes->ver = 1;
>>> +	inodes->compat = 1;
>>> +	inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
>>> +	inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
>>> +	inodes->total = cpu_to_le64(sum);
>>> +	items++;
>>> +
>>>    	put_unaligned_le32(items, &head->num);
>>>    	msg->front.iov_len = len;
>>>    	msg->hdr.version = cpu_to_le16(1);
>>> diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
>>> index af6038ff39d4..4ceb462135d7 100644
>>> --- a/fs/ceph/metric.h
>>> +++ b/fs/ceph/metric.h
>>> @@ -14,8 +14,11 @@ enum ceph_metric_type {
>>>    	CLIENT_METRIC_TYPE_WRITE_LATENCY,
>>>    	CLIENT_METRIC_TYPE_METADATA_LATENCY,
>>>    	CLIENT_METRIC_TYPE_DENTRY_LEASE,
>>> +	CLIENT_METRIC_TYPE_OPENED_FILES,
>>> +	CLIENT_METRIC_TYPE_PINNED_ICAPS,
>>> +	CLIENT_METRIC_TYPE_OPENED_INODES,
>>>    
>>>
>>> -	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
>>> +	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
>>>    };
>>>    
>>>
>>>    /*
>>> @@ -28,6 +31,9 @@ enum ceph_metric_type {
>>>    	CLIENT_METRIC_TYPE_WRITE_LATENCY,	\
>>>    	CLIENT_METRIC_TYPE_METADATA_LATENCY,	\
>>>    	CLIENT_METRIC_TYPE_DENTRY_LEASE,	\
>>> +	CLIENT_METRIC_TYPE_OPENED_FILES,	\
>>> +	CLIENT_METRIC_TYPE_PINNED_ICAPS,	\
>>> +	CLIENT_METRIC_TYPE_OPENED_INODES,	\
>>>    						\
>>>    	CLIENT_METRIC_TYPE_MAX,			\
>>>    }
>>> @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
>>>    	__le64 total;
>>>    } __packed;
>>>    
>>>
>>> +/* metric opened files header */
>>> +struct ceph_opened_files {
>>> +	__le32 type;     /* ceph metric type */
>>> +
>>> +	__u8  ver;
>>> +	__u8  compat;
>>> +
>>> +	__le32 data_len; /* length of sizeof(opened_files + total) */
>>> +	__le64 opened_files;
>>> +	__le64 total;
>>> +} __packed;
>>> +
>>> +/* metric pinned i_caps header */
>>> +struct ceph_pinned_icaps {
>>> +	__le32 type;     /* ceph metric type */
>>> +
>>> +	__u8  ver;
>>> +	__u8  compat;
>>> +
>>> +	__le32 data_len; /* length of sizeof(pinned_icaps + total) */
>>> +	__le64 pinned_icaps;
>>> +	__le64 total;
>>> +} __packed;
>>> +
>>> +/* metric opened inodes header */
>>> +struct ceph_opened_inodes {
>>> +	__le32 type;     /* ceph metric type */
>>> +
>>> +	__u8  ver;
>>> +	__u8  compat;
>>> +
>>> +	__le32 data_len; /* length of sizeof(opened_inodes + total) */
>>> +	__le64 opened_inodes;
>>> +	__le64 total;
>>> +} __packed;
>>> +
>>>    struct ceph_metric_head {
>>>    	__le32 num;	/* the number of metrics that will be sent */
>>>    } __packed;
>>


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
  2021-03-22 12:32     ` Xiubo Li
@ 2021-03-23 19:51       ` Jeff Layton
  0 siblings, 0 replies; 7+ messages in thread
From: Jeff Layton @ 2021-03-23 19:51 UTC (permalink / raw)
  To: Xiubo Li; +Cc: pdonnell, ceph-devel, Ilya Dryomov

On Mon, 2021-03-22 at 20:32 +0800, Xiubo Li wrote:
> On 2021/3/22 19:55, Jeff Layton wrote:
> > Oh! I hadn't realized that this patch was still unmerged.
> 
> No worry, this patch was waiting the MDS side patch, which was just 
> merged last week :-)
> 
> I almost forgot this patch too before I saw the ceph tracker today.
> 
> > Merged into testing branch. I'd have pulled this into testing a while
> > ago if I had realized. Let me know if I miss any in the future (or am
> > missing any now).
> 
> Sure, np.
> 
> BRs
> 
> Xiubo
> 

After doing some testing with this patch, I hit the following KASAN pop
today. I've backed this patch out of the testing branch for now:

[  110.969263] ==================================================================
[  110.972933] BUG: KASAN: slab-out-of-bounds in metric_delayed_work+0x5e9/0x880 [ceph]
[  110.976096] Write of size 8 at addr ffff88811b76ab90 by task kworker/7:2/613
[  110.978972] 
[  110.979668] CPU: 7 PID: 613 Comm: kworker/7:2 Tainted: G            E     5.12.0-rc2+ #70
[  110.983015] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.14.0-1.fc33 04/01/2014
[  110.986467] Workqueue: events metric_delayed_work [ceph]
[  110.988803] Call Trace:
[  110.989869]  dump_stack+0xa5/0xdc
[  110.991297]  print_address_description.constprop.0+0x18/0x160
[  110.993659]  ? metric_delayed_work+0x5e9/0x880 [ceph]
[  110.995892]  ? metric_delayed_work+0x5e9/0x880 [ceph]
[  110.998108]  kasan_report.cold+0x7f/0x111
[  110.999818]  ? _raw_spin_lock_irq+0x61/0x90
[  111.001563]  ? metric_delayed_work+0x5e9/0x880 [ceph]
[  111.003814]  kasan_check_range+0xf9/0x1e0
[  111.005503]  metric_delayed_work+0x5e9/0x880 [ceph]
[  111.007688]  ? ceph_caps_for_mode+0x40/0x40 [ceph]
[  111.009855]  process_one_work+0x525/0x9b0
[  111.011597]  ? pwq_dec_nr_in_flight+0x110/0x110
[  111.013504]  ? lock_acquired+0x2fe/0x560
[  111.015230]  worker_thread+0x2ea/0x6e0
[  111.016840]  ? __kthread_parkme+0xc0/0xe0
[  111.018534]  ? process_one_work+0x9b0/0x9b0
[  111.020262]  kthread+0x1fb/0x220
[  111.021622]  ? __kthread_bind_mask+0x70/0x70
[  111.023423]  ret_from_fork+0x22/0x30
[  111.025004] 
[  111.025687] Allocated by task 613:
[  111.027105]  kasan_save_stack+0x1b/0x40
[  111.028740]  __kasan_kmalloc+0x7a/0x90
[  111.029583]  ceph_kvmalloc+0xc7/0x110 [libceph]
[  111.030611]  ceph_msg_new2+0x15d/0x270 [libceph]
[  111.031537]  metric_delayed_work+0x1c5/0x880 [ceph]
[  111.032414]  process_one_work+0x525/0x9b0
[  111.033093]  worker_thread+0x2ea/0x6e0
[  111.033763]  kthread+0x1fb/0x220
[  111.034329]  ret_from_fork+0x22/0x30
[  111.038203] 
[  111.041730] The buggy address belongs to the object at ffff88811b76ab00
[  111.041730]  which belongs to the cache kmalloc-192 of size 192
[  111.050535] The buggy address is located 144 bytes inside of
[  111.050535]  192-byte region [ffff88811b76ab00, ffff88811b76abc0)
[  111.059194] The buggy address belongs to the page:
[  111.063348] page:0000000052971d41 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x11b76a
[  111.068337] head:0000000052971d41 order:1 compound_mapcount:0
[  111.073364] flags: 0x17ffffc0010200(slab|head)
[  111.078020] raw: 0017ffffc0010200 0000000000000000 0000000100000001 ffff888100042a00
[  111.083177] raw: 0000000000000000 0000000080200020 00000001ffffffff 0000000000000000
[  111.088350] page dumped because: kasan: bad access detected
[  111.093064] 
[  111.096773] Memory state around the buggy address:
[  111.101168]  ffff88811b76aa80: 00 00 fc fc fc fc fc fc fc fc fc fc fc fc fc fc
[  111.106154]  ffff88811b76ab00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
[  111.111073] >ffff88811b76ab80: 00 00 06 fc fc fc fc fc fc fc fc fc fc fc fc fc
[  111.115973]                          ^
[  111.120219]  ffff88811b76ac00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
[  111.125263]  ffff88811b76ac80: 00 fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
[  111.130227] ==================================================================

-- 
Jeff Layton <jlayton@kernel.org>


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
  2020-11-26  3:47 [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon xiubli
  2021-03-22  2:30 ` Xiubo Li
@ 2021-03-23 19:52 ` Jeff Layton
  1 sibling, 0 replies; 7+ messages in thread
From: Jeff Layton @ 2021-03-23 19:52 UTC (permalink / raw)
  To: xiubli, idryomov; +Cc: zyan, pdonnell, ceph-devel

On Thu, 2020-11-26 at 11:47 +0800, xiubli@redhat.com wrote:
> From: Xiubo Li <xiubli@redhat.com>
> 
> For the old ceph version, if it received this metric message containing
> the send opened files/pinned caps/opened inodes metric info, it will
> just ignore them.
> 
> URL: https://tracker.ceph.com/issues/46866
> Signed-off-by: Xiubo Li <xiubli@redhat.com>
> ---
>  fs/ceph/metric.c | 38 +++++++++++++++++++++++++++++++++++++-
>  fs/ceph/metric.h | 44 +++++++++++++++++++++++++++++++++++++++++++-
>  2 files changed, 80 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> index 5ec94bd4c1de..306bd599d940 100644
> --- a/fs/ceph/metric.c
> +++ b/fs/ceph/metric.c
> @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>  	struct ceph_metric_write_latency *write;
>  	struct ceph_metric_metadata_latency *meta;
>  	struct ceph_metric_dlease *dlease;
> +	struct ceph_opened_files *files;
> +	struct ceph_pinned_icaps *icaps;
> +	struct ceph_opened_inodes *inodes;
>  	struct ceph_client_metric *m = &mdsc->metric;
>  	u64 nr_caps = atomic64_read(&m->total_caps);
>  	struct ceph_msg *msg;
> @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>  	s32 len;
>  
>  	len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
> -	      + sizeof(*meta) + sizeof(*dlease);
> +	      + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + sizeof(icaps)
> +	      + sizeof(inodes);
>  

These sizeof values look wrong. The buffer requires more than pointers
for those. You probably want:

 ... + sizeof(*files) + sizeof(*icaps) + sizeof(*inodes);

>  	msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
>  	if (!msg) {
> @@ -95,6 +99,38 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
>  	dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
>  	items++;
>  
> +	sum = percpu_counter_sum(&m->total_inodes);
> +
> +	/* encode the opened files metric */
> +	files = (struct ceph_opened_files *)(dlease + 1);
> +	files->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES);
> +	files->ver = 1;
> +	files->compat = 1;
> +	files->data_len = cpu_to_le32(sizeof(*files) - 10);
> +	files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files));
> +	files->total = cpu_to_le64(sum);
> +	items++;
> +
> +	/* encode the pinned icaps metric */
> +	icaps = (struct ceph_pinned_icaps *)(files + 1);
> +	icaps->type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS);
> +	icaps->ver = 1;
> +	icaps->compat = 1;
> +	icaps->data_len = cpu_to_le32(sizeof(*icaps) - 10);
> +	icaps->pinned_icaps = cpu_to_le64(nr_caps);
> +	icaps->total = cpu_to_le64(sum);
> +	items++;
> +
> +	/* encode the opened inodes metric */
> +	inodes = (struct ceph_opened_inodes *)(icaps + 1);
> +	inodes->type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES);
> +	inodes->ver = 1;
> +	inodes->compat = 1;
> +	inodes->data_len = cpu_to_le32(sizeof(*inodes) - 10);
> +	inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes));
> +	inodes->total = cpu_to_le64(sum);
> +	items++;
> +
>  	put_unaligned_le32(items, &head->num);
>  	msg->front.iov_len = len;
>  	msg->hdr.version = cpu_to_le16(1);
> diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
> index af6038ff39d4..4ceb462135d7 100644
> --- a/fs/ceph/metric.h
> +++ b/fs/ceph/metric.h
> @@ -14,8 +14,11 @@ enum ceph_metric_type {
>  	CLIENT_METRIC_TYPE_WRITE_LATENCY,
>  	CLIENT_METRIC_TYPE_METADATA_LATENCY,
>  	CLIENT_METRIC_TYPE_DENTRY_LEASE,
> +	CLIENT_METRIC_TYPE_OPENED_FILES,
> +	CLIENT_METRIC_TYPE_PINNED_ICAPS,
> +	CLIENT_METRIC_TYPE_OPENED_INODES,
>  
> -	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
> +	CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_OPENED_INODES,
>  };
>  
>  /*
> @@ -28,6 +31,9 @@ enum ceph_metric_type {
>  	CLIENT_METRIC_TYPE_WRITE_LATENCY,	\
>  	CLIENT_METRIC_TYPE_METADATA_LATENCY,	\
>  	CLIENT_METRIC_TYPE_DENTRY_LEASE,	\
> +	CLIENT_METRIC_TYPE_OPENED_FILES,	\
> +	CLIENT_METRIC_TYPE_PINNED_ICAPS,	\
> +	CLIENT_METRIC_TYPE_OPENED_INODES,	\
>  						\
>  	CLIENT_METRIC_TYPE_MAX,			\
>  }
> @@ -94,6 +100,42 @@ struct ceph_metric_dlease {
>  	__le64 total;
>  } __packed;
>  
> +/* metric opened files header */
> +struct ceph_opened_files {
> +	__le32 type;     /* ceph metric type */
> +
> +	__u8  ver;
> +	__u8  compat;
> +
> +	__le32 data_len; /* length of sizeof(opened_files + total) */
> +	__le64 opened_files;
> +	__le64 total;
> +} __packed;
> +
> +/* metric pinned i_caps header */
> +struct ceph_pinned_icaps {
> +	__le32 type;     /* ceph metric type */
> +
> +	__u8  ver;
> +	__u8  compat;
> +
> +	__le32 data_len; /* length of sizeof(pinned_icaps + total) */
> +	__le64 pinned_icaps;
> +	__le64 total;
> +} __packed;
> +
> +/* metric opened inodes header */
> +struct ceph_opened_inodes {
> +	__le32 type;     /* ceph metric type */
> +
> +	__u8  ver;
> +	__u8  compat;
> +
> +	__le32 data_len; /* length of sizeof(opened_inodes + total) */
> +	__le64 opened_inodes;
> +	__le64 total;
> +} __packed;
> +
>  struct ceph_metric_head {
>  	__le32 num;	/* the number of metrics that will be sent */
>  } __packed;

-- 
Jeff Layton <jlayton@kernel.org>


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon
  2021-03-22 11:55   ` Jeff Layton
  2021-03-22 12:32     ` Xiubo Li
@ 2021-03-24  4:54     ` Xiubo Li
  1 sibling, 0 replies; 7+ messages in thread
From: Xiubo Li @ 2021-03-24  4:54 UTC (permalink / raw)
  To: Jeff Layton; +Cc: pdonnell, ceph-devel, Ilya Dryomov

On 2021/3/24 12:53, Xiubo Li wrote:

[...]

> > diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
> > index 5ec94bd4c1de..306bd599d940 100644
> > --- a/fs/ceph/metric.c
> > +++ b/fs/ceph/metric.c
> > @@ -17,6 +17,9 @@ static bool ceph_mdsc_send_metrics(struct 
> ceph_mds_client *mdsc,
> >      struct ceph_metric_write_latency *write;
> >      struct ceph_metric_metadata_latency *meta;
> >      struct ceph_metric_dlease *dlease;
> > +    struct ceph_opened_files *files;
> > +    struct ceph_pinned_icaps *icaps;
> > +    struct ceph_opened_inodes *inodes;
> >      struct ceph_client_metric *m = &mdsc->metric;
> >      u64 nr_caps = atomic64_read(&m->total_caps);
> >      struct ceph_msg *msg;
> > @@ -26,7 +29,8 @@ static bool ceph_mdsc_send_metrics(struct 
> ceph_mds_client *mdsc,
> >      s32 len;
> >
> >      len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + 
> sizeof(*write)
> > -          + sizeof(*meta) + sizeof(*dlease);
> > +          + sizeof(*meta) + sizeof(*dlease) + sizeof(files) + 
> sizeof(icaps)
> > +          + sizeof(inodes);
> >
>
> These sizeof values look wrong. The buffer requires more than pointers
> for those. You probably want:
>
>  ... + sizeof(*files) + sizeof(*icaps) + sizeof(*inodes);
>
Yeah, will fix it.

Thanks



^ permalink raw reply	[flat|nested] 7+ messages in thread

end of thread, other threads:[~2021-03-24  4:56 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-11-26  3:47 [PATCH] ceph: send opened files/pinned caps/opened inodes metrics to MDS daemon xiubli
2021-03-22  2:30 ` Xiubo Li
2021-03-22 11:55   ` Jeff Layton
2021-03-22 12:32     ` Xiubo Li
2021-03-23 19:51       ` Jeff Layton
2021-03-24  4:54     ` Xiubo Li
2021-03-23 19:52 ` Jeff Layton

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).