All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] mm: throttle show_mem from warn_alloc
@ 2016-12-15 10:15 ` Michal Hocko
  0 siblings, 0 replies; 6+ messages in thread
From: Michal Hocko @ 2016-12-15 10:15 UTC (permalink / raw)
  To: Andrew Morton; +Cc: Tetsuo Handa, linux-mm, LKML, Michal Hocko

From: Michal Hocko <mhocko@suse.com>

Tetsuo has been stressing OOM killer path with many parallel allocation
requests when he has noticed that it is not all that hard to swamp
kernel logs with warn_alloc messages caused by allocation stalls. Even
though the allocation stall message is triggered only once in 10s there
might be many different tasks hitting it roughly around the same time.

A big part of the output is show_mem() which can generate a lot of
output even on a small machines. There is no reason to show the state of
memory counter for each allocation stall, especially when multiple of
them are reported in a short time period. Chances are that not much has
changed since the last report. This patch simply rate limits show_mem
called from warn_alloc to only dump something once per second. This
should be enough to give us a clue why an allocation might be stalling
while burst of warnings will not swamp log with too much data.

While we are at it, extract all the show_mem related handling (filters)
into a separate function warn_alloc_show_mem. This will make the code
cleaner and as a bonus point we can distinguish which part of warn_alloc
got throttled due to rate limiting as ___ratelimit dumps the caller.

Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Signed-off-by: Michal Hocko <mhocko@suse.com>
---
 mm/page_alloc.c | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3f2c9e535f7f..7a46fc300f18 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3019,14 +3019,13 @@ static DEFINE_RATELIMIT_STATE(nopage_rs,
 		DEFAULT_RATELIMIT_INTERVAL,
 		DEFAULT_RATELIMIT_BURST);
 
-void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
+static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
+
+static void warn_alloc_show_mem(gfp_t gfp_mask)
 {
 	unsigned int filter = SHOW_MEM_FILTER_NODES;
-	struct va_format vaf;
-	va_list args;
 
-	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
-	    debug_guardpage_minorder() > 0)
+	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
 		return;
 
 	/*
@@ -3041,6 +3040,18 @@ void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
 	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
 		filter &= ~SHOW_MEM_FILTER_NODES;
 
+	show_mem(filter);
+}
+
+void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
+	    debug_guardpage_minorder() > 0)
+		return;
+
 	pr_warn("%s: ", current->comm);
 
 	va_start(args, fmt);
@@ -3052,8 +3063,7 @@ void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
 	pr_cont(", mode:%#x(%pGg)\n", gfp_mask, &gfp_mask);
 
 	dump_stack();
-	if (!should_suppress_show_mem())
-		show_mem(filter);
+	warn_alloc_show_mem(gfp_mask);
 }
 
 static inline struct page *
-- 
2.10.2

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [PATCH] mm: throttle show_mem from warn_alloc
@ 2016-12-15 10:15 ` Michal Hocko
  0 siblings, 0 replies; 6+ messages in thread
From: Michal Hocko @ 2016-12-15 10:15 UTC (permalink / raw)
  To: Andrew Morton; +Cc: Tetsuo Handa, linux-mm, LKML, Michal Hocko

From: Michal Hocko <mhocko@suse.com>

Tetsuo has been stressing OOM killer path with many parallel allocation
requests when he has noticed that it is not all that hard to swamp
kernel logs with warn_alloc messages caused by allocation stalls. Even
though the allocation stall message is triggered only once in 10s there
might be many different tasks hitting it roughly around the same time.

A big part of the output is show_mem() which can generate a lot of
output even on a small machines. There is no reason to show the state of
memory counter for each allocation stall, especially when multiple of
them are reported in a short time period. Chances are that not much has
changed since the last report. This patch simply rate limits show_mem
called from warn_alloc to only dump something once per second. This
should be enough to give us a clue why an allocation might be stalling
while burst of warnings will not swamp log with too much data.

While we are at it, extract all the show_mem related handling (filters)
into a separate function warn_alloc_show_mem. This will make the code
cleaner and as a bonus point we can distinguish which part of warn_alloc
got throttled due to rate limiting as ___ratelimit dumps the caller.

Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Signed-off-by: Michal Hocko <mhocko@suse.com>
---
 mm/page_alloc.c | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3f2c9e535f7f..7a46fc300f18 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3019,14 +3019,13 @@ static DEFINE_RATELIMIT_STATE(nopage_rs,
 		DEFAULT_RATELIMIT_INTERVAL,
 		DEFAULT_RATELIMIT_BURST);
 
-void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
+static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
+
+static void warn_alloc_show_mem(gfp_t gfp_mask)
 {
 	unsigned int filter = SHOW_MEM_FILTER_NODES;
-	struct va_format vaf;
-	va_list args;
 
-	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
-	    debug_guardpage_minorder() > 0)
+	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
 		return;
 
 	/*
@@ -3041,6 +3040,18 @@ void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
 	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
 		filter &= ~SHOW_MEM_FILTER_NODES;
 
+	show_mem(filter);
+}
+
+void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
+	    debug_guardpage_minorder() > 0)
+		return;
+
 	pr_warn("%s: ", current->comm);
 
 	va_start(args, fmt);
@@ -3052,8 +3063,7 @@ void warn_alloc(gfp_t gfp_mask, const char *fmt, ...)
 	pr_cont(", mode:%#x(%pGg)\n", gfp_mask, &gfp_mask);
 
 	dump_stack();
-	if (!should_suppress_show_mem())
-		show_mem(filter);
+	warn_alloc_show_mem(gfp_mask);
 }
 
 static inline struct page *
-- 
2.10.2

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH] mm: throttle show_mem from warn_alloc
  2016-12-15 10:15 ` Michal Hocko
@ 2016-12-19 23:21   ` Andrew Morton
  -1 siblings, 0 replies; 6+ messages in thread
From: Andrew Morton @ 2016-12-19 23:21 UTC (permalink / raw)
  To: Michal Hocko; +Cc: Tetsuo Handa, linux-mm, LKML, Michal Hocko

On Thu, 15 Dec 2016 11:15:10 +0100 Michal Hocko <mhocko@kernel.org> wrote:

> Tetsuo has been stressing OOM killer path with many parallel allocation
> requests when he has noticed that it is not all that hard to swamp
> kernel logs with warn_alloc messages caused by allocation stalls. Even
> though the allocation stall message is triggered only once in 10s there
> might be many different tasks hitting it roughly around the same time.
> 
> A big part of the output is show_mem() which can generate a lot of
> output even on a small machines. There is no reason to show the state of
> memory counter for each allocation stall, especially when multiple of
> them are reported in a short time period. Chances are that not much has
> changed since the last report. This patch simply rate limits show_mem
> called from warn_alloc to only dump something once per second. This
> should be enough to give us a clue why an allocation might be stalling
> while burst of warnings will not swamp log with too much data.
> 
> While we are at it, extract all the show_mem related handling (filters)
> into a separate function warn_alloc_show_mem. This will make the code
> cleaner and as a bonus point we can distinguish which part of warn_alloc
> got throttled due to rate limiting as ___ratelimit dumps the caller.

These guys don't need file-wide scope...

--- a/mm/page_alloc.c~mm-throttle-show_mem-from-warn_alloc-fix
+++ a/mm/page_alloc.c
@@ -3018,15 +3018,10 @@ static inline bool should_suppress_show_
 	return ret;
 }
 
-static DEFINE_RATELIMIT_STATE(nopage_rs,
-		DEFAULT_RATELIMIT_INTERVAL,
-		DEFAULT_RATELIMIT_BURST);
-
-static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
-
 static void warn_alloc_show_mem(gfp_t gfp_mask)
 {
 	unsigned int filter = SHOW_MEM_FILTER_NODES;
+	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
 
 	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
 		return;
@@ -3050,6 +3045,8 @@ void warn_alloc(gfp_t gfp_mask, const ch
 {
 	struct va_format vaf;
 	va_list args;
+	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
+				      DEFAULT_RATELIMIT_BURST);
 
 	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
 	    debug_guardpage_minorder() > 0)
_

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH] mm: throttle show_mem from warn_alloc
@ 2016-12-19 23:21   ` Andrew Morton
  0 siblings, 0 replies; 6+ messages in thread
From: Andrew Morton @ 2016-12-19 23:21 UTC (permalink / raw)
  To: Michal Hocko; +Cc: Tetsuo Handa, linux-mm, LKML, Michal Hocko

On Thu, 15 Dec 2016 11:15:10 +0100 Michal Hocko <mhocko@kernel.org> wrote:

> Tetsuo has been stressing OOM killer path with many parallel allocation
> requests when he has noticed that it is not all that hard to swamp
> kernel logs with warn_alloc messages caused by allocation stalls. Even
> though the allocation stall message is triggered only once in 10s there
> might be many different tasks hitting it roughly around the same time.
> 
> A big part of the output is show_mem() which can generate a lot of
> output even on a small machines. There is no reason to show the state of
> memory counter for each allocation stall, especially when multiple of
> them are reported in a short time period. Chances are that not much has
> changed since the last report. This patch simply rate limits show_mem
> called from warn_alloc to only dump something once per second. This
> should be enough to give us a clue why an allocation might be stalling
> while burst of warnings will not swamp log with too much data.
> 
> While we are at it, extract all the show_mem related handling (filters)
> into a separate function warn_alloc_show_mem. This will make the code
> cleaner and as a bonus point we can distinguish which part of warn_alloc
> got throttled due to rate limiting as ___ratelimit dumps the caller.

These guys don't need file-wide scope...

--- a/mm/page_alloc.c~mm-throttle-show_mem-from-warn_alloc-fix
+++ a/mm/page_alloc.c
@@ -3018,15 +3018,10 @@ static inline bool should_suppress_show_
 	return ret;
 }
 
-static DEFINE_RATELIMIT_STATE(nopage_rs,
-		DEFAULT_RATELIMIT_INTERVAL,
-		DEFAULT_RATELIMIT_BURST);
-
-static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
-
 static void warn_alloc_show_mem(gfp_t gfp_mask)
 {
 	unsigned int filter = SHOW_MEM_FILTER_NODES;
+	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
 
 	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
 		return;
@@ -3050,6 +3045,8 @@ void warn_alloc(gfp_t gfp_mask, const ch
 {
 	struct va_format vaf;
 	va_list args;
+	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
+				      DEFAULT_RATELIMIT_BURST);
 
 	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
 	    debug_guardpage_minorder() > 0)
_

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH] mm: throttle show_mem from warn_alloc
  2016-12-19 23:21   ` Andrew Morton
@ 2016-12-20  8:35     ` Michal Hocko
  -1 siblings, 0 replies; 6+ messages in thread
From: Michal Hocko @ 2016-12-20  8:35 UTC (permalink / raw)
  To: Andrew Morton; +Cc: Tetsuo Handa, linux-mm, LKML

On Mon 19-12-16 15:21:25, Andrew Morton wrote:
> On Thu, 15 Dec 2016 11:15:10 +0100 Michal Hocko <mhocko@kernel.org> wrote:
> 
> > Tetsuo has been stressing OOM killer path with many parallel allocation
> > requests when he has noticed that it is not all that hard to swamp
> > kernel logs with warn_alloc messages caused by allocation stalls. Even
> > though the allocation stall message is triggered only once in 10s there
> > might be many different tasks hitting it roughly around the same time.
> > 
> > A big part of the output is show_mem() which can generate a lot of
> > output even on a small machines. There is no reason to show the state of
> > memory counter for each allocation stall, especially when multiple of
> > them are reported in a short time period. Chances are that not much has
> > changed since the last report. This patch simply rate limits show_mem
> > called from warn_alloc to only dump something once per second. This
> > should be enough to give us a clue why an allocation might be stalling
> > while burst of warnings will not swamp log with too much data.
> > 
> > While we are at it, extract all the show_mem related handling (filters)
> > into a separate function warn_alloc_show_mem. This will make the code
> > cleaner and as a bonus point we can distinguish which part of warn_alloc
> > got throttled due to rate limiting as ___ratelimit dumps the caller.
> 
> These guys don't need file-wide scope...
> 
> --- a/mm/page_alloc.c~mm-throttle-show_mem-from-warn_alloc-fix
> +++ a/mm/page_alloc.c
> @@ -3018,15 +3018,10 @@ static inline bool should_suppress_show_
>  	return ret;
>  }
>  
> -static DEFINE_RATELIMIT_STATE(nopage_rs,
> -		DEFAULT_RATELIMIT_INTERVAL,
> -		DEFAULT_RATELIMIT_BURST);
> -
> -static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
> -
>  static void warn_alloc_show_mem(gfp_t gfp_mask)
>  {
>  	unsigned int filter = SHOW_MEM_FILTER_NODES;
> +	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
>  
>  	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
>  		return;
> @@ -3050,6 +3045,8 @@ void warn_alloc(gfp_t gfp_mask, const ch
>  {
>  	struct va_format vaf;
>  	va_list args;
> +	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
> +				      DEFAULT_RATELIMIT_BURST);
>  
>  	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
>  	    debug_guardpage_minorder() > 0)
> _
> 

Acked-by: Michal Hocko <mhocko@suse.com>

-- 
Michal Hocko
SUSE Labs

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [PATCH] mm: throttle show_mem from warn_alloc
@ 2016-12-20  8:35     ` Michal Hocko
  0 siblings, 0 replies; 6+ messages in thread
From: Michal Hocko @ 2016-12-20  8:35 UTC (permalink / raw)
  To: Andrew Morton; +Cc: Tetsuo Handa, linux-mm, LKML

On Mon 19-12-16 15:21:25, Andrew Morton wrote:
> On Thu, 15 Dec 2016 11:15:10 +0100 Michal Hocko <mhocko@kernel.org> wrote:
> 
> > Tetsuo has been stressing OOM killer path with many parallel allocation
> > requests when he has noticed that it is not all that hard to swamp
> > kernel logs with warn_alloc messages caused by allocation stalls. Even
> > though the allocation stall message is triggered only once in 10s there
> > might be many different tasks hitting it roughly around the same time.
> > 
> > A big part of the output is show_mem() which can generate a lot of
> > output even on a small machines. There is no reason to show the state of
> > memory counter for each allocation stall, especially when multiple of
> > them are reported in a short time period. Chances are that not much has
> > changed since the last report. This patch simply rate limits show_mem
> > called from warn_alloc to only dump something once per second. This
> > should be enough to give us a clue why an allocation might be stalling
> > while burst of warnings will not swamp log with too much data.
> > 
> > While we are at it, extract all the show_mem related handling (filters)
> > into a separate function warn_alloc_show_mem. This will make the code
> > cleaner and as a bonus point we can distinguish which part of warn_alloc
> > got throttled due to rate limiting as ___ratelimit dumps the caller.
> 
> These guys don't need file-wide scope...
> 
> --- a/mm/page_alloc.c~mm-throttle-show_mem-from-warn_alloc-fix
> +++ a/mm/page_alloc.c
> @@ -3018,15 +3018,10 @@ static inline bool should_suppress_show_
>  	return ret;
>  }
>  
> -static DEFINE_RATELIMIT_STATE(nopage_rs,
> -		DEFAULT_RATELIMIT_INTERVAL,
> -		DEFAULT_RATELIMIT_BURST);
> -
> -static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
> -
>  static void warn_alloc_show_mem(gfp_t gfp_mask)
>  {
>  	unsigned int filter = SHOW_MEM_FILTER_NODES;
> +	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
>  
>  	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
>  		return;
> @@ -3050,6 +3045,8 @@ void warn_alloc(gfp_t gfp_mask, const ch
>  {
>  	struct va_format vaf;
>  	va_list args;
> +	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
> +				      DEFAULT_RATELIMIT_BURST);
>  
>  	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
>  	    debug_guardpage_minorder() > 0)
> _
> 

Acked-by: Michal Hocko <mhocko@suse.com>

-- 
Michal Hocko
SUSE Labs

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2016-12-20  8:35 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-12-15 10:15 [PATCH] mm: throttle show_mem from warn_alloc Michal Hocko
2016-12-15 10:15 ` Michal Hocko
2016-12-19 23:21 ` Andrew Morton
2016-12-19 23:21   ` Andrew Morton
2016-12-20  8:35   ` Michal Hocko
2016-12-20  8:35     ` Michal Hocko

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.