[PATCH v2 0/8] thread_info cleanups and stack caching

[PATCH v2 0/8] thread_info cleanups and stack caching

[Andy Lutomirski]

This is the last bit of the vmap stack pile.  Now that thread_info
is non-magical, we can free the thread stack as soon as the task is
dead (without waiting for RCU) and then, if vmapped stacks are in
use, cache the entire stack for reuse on the same cpu.

This seems to be an overall speedup of about 0.5-1 µs per
pthread_create/join compared to the old CONFIG_VMAP_STACK=n baseline
in a simple test -- a percpu cache of vmalloced stacks appears to be
a bit faster than a high-order stack allocation, at least when the
cache hits.  (I expect that workloads with a low cache hit rate are
likely to be dominated by other effects anyway.)

Changes from v1:
 - Rebased.
 - Added a comment fixup (patch 1).
 - Add one more try_get_task_stack() that Josh noticed.

Changes from before:
 - A bunch of the series is already in 4.8-rc.
 - Added the get_wchan() and collect_syscall() patches.
 - Rebased.

Andy Lutomirski (7):
  x86/entry/64: Fix a minor comment rebase error
  sched: Add try_get_task_stack() and put_task_stack()
  x86/dumpstack: Pin the target stack when dumping it
  x86/process: Pin the target stack in get_wchan()
  lib/syscall: Pin the task stack in collect_syscall()
  sched: Free the stack early if CONFIG_THREAD_INFO_IN_TASK
  fork: Cache two thread stacks per cpu if CONFIG_VMAP_STACK is set

Oleg Nesterov (1):
  kthread: to_live_kthread() needs try_get_task_stack()

 arch/x86/entry/entry_64.S      |  1 -
 arch/x86/kernel/dumpstack_32.c |  5 +++
 arch/x86/kernel/dumpstack_64.c |  5 +++
 arch/x86/kernel/process.c      | 22 +++++++---
 arch/x86/kernel/stacktrace.c   |  5 +++
 include/linux/init_task.h      |  4 +-
 include/linux/sched.h          | 30 +++++++++++++
 init/Kconfig                   |  3 ++
 kernel/fork.c                  | 97 +++++++++++++++++++++++++++++++++++++-----
 kernel/kthread.c               |  8 +++-
 kernel/sched/core.c            |  4 ++
 lib/syscall.c                  | 15 ++++++-
 12 files changed, 176 insertions(+), 23 deletions(-)

-- 
2.7.4

[PATCH v2 1/8] x86/entry/64: Fix a minor comment rebase error

[Andy Lutomirski]

When I rebased my thread_info changes onto Brian's switch_to()
changes, I carefully checked that I fixed up all the code correctly,
but I missed a comment :(

Fixes: 15f4eae70d36 ("x86: Move thread_info into task_struct")
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 arch/x86/entry/entry_64.S | 1 -
 1 file changed, 1 deletion(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 2b46384b4a4f..80ab68a42621 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -372,7 +372,6 @@ END(ptregs_\func)
 /*
  * %rdi: prev task
  * %rsi: next task
- * rsi: task we're switching to
  */
 ENTRY(__switch_to_asm)
 	/*
-- 
2.7.4

[tip:x86/asm] x86/entry/64: Fix a minor comment rebase error

[tip-bot for Andy Lutomirski]

Commit-ID:  ff0071c03684485495e06f3936399eb9c93141a6
Gitweb:     http://git.kernel.org/tip/ff0071c03684485495e06f3936399eb9c93141a6
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:42 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:52 +0200

x86/entry/64: Fix a minor comment rebase error

When I rebased my thread_info changes onto Brian's switch_to()
changes, I carefully checked that I fixed up all the code correctly,
but I missed a comment :(

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Fixes: 15f4eae70d36 ("x86: Move thread_info into task_struct")
Link: http://lkml.kernel.org/r/089fe1e1cbe8b258b064fccbb1a5a5fd23861031.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/entry/entry_64.S | 1 -
 1 file changed, 1 deletion(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 2b46384..80ab68a 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -372,7 +372,6 @@ END(ptregs_\func)
 /*
  * %rdi: prev task
  * %rsi: next task
- * rsi: task we're switching to
  */
 ENTRY(__switch_to_asm)
 	/*

[PATCH v2 2/8] sched: Add try_get_task_stack() and put_task_stack()

[Andy Lutomirski]

There are a few places in the kernel that access stack memory
belonging to a different task.  Before we can start freeing task
stacks before the task_struct is freed, we need a way for those code
paths to pin the stack.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 include/linux/sched.h | 16 ++++++++++++++++
 init/Kconfig          |  3 +++
 2 files changed, 19 insertions(+)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index a287e8b13549..a95867267e9f 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -3094,11 +3094,19 @@ static inline struct thread_info *task_thread_info(struct task_struct *task)
 {
 	return &task->thread_info;
 }
+
+/*
+ * When accessing the stack of a non-current task that might exit, use
+ * try_get_task_stack() instead.  task_stack_page will return a pointer
+ * that could get freed out from under you.
+ */
 static inline void *task_stack_page(const struct task_struct *task)
 {
 	return task->stack;
 }
+
 #define setup_thread_stack(new,old)	do { } while(0)
+
 static inline unsigned long *end_of_stack(const struct task_struct *task)
 {
 	return task->stack;
@@ -3134,6 +3142,14 @@ static inline unsigned long *end_of_stack(struct task_struct *p)
 }
 
 #endif
+
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+	return task_stack_page(tsk);
+}
+
+static inline void put_task_stack(struct task_struct *tsk) {}
+
 #define task_stack_end_corrupted(task) \
 		(*(end_of_stack(task)) != STACK_END_MAGIC)
 
diff --git a/init/Kconfig b/init/Kconfig
index ec8d43894b02..3b9a47fe843b 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -33,6 +33,9 @@ config THREAD_INFO_IN_TASK
 	  make this work, an arch will need to remove all thread_info fields
 	  except flags and fix any runtime bugs.
 
+	  One subtle change that will be needed is to use try_get_task_stack()
+	  and put_task_stack() in save_thread_stack_tsk() and get_wchan().
+
 menu "General setup"
 
 config BROKEN
-- 
2.7.4

[tip:x86/asm] sched/core: Add try_get_task_stack() and put_task_stack()

[tip-bot for Andy Lutomirski]

Commit-ID:  c6c314a613cd7d03fb97713e0d642b493de42e69
Gitweb:     http://git.kernel.org/tip/c6c314a613cd7d03fb97713e0d642b493de42e69
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:43 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:53 +0200

sched/core: Add try_get_task_stack() and put_task_stack()

There are a few places in the kernel that access stack memory
belonging to a different task.  Before we can start freeing task
stacks before the task_struct is freed, we need a way for those code
paths to pin the stack.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/17a434f50ad3d77000104f21666575e10a9c1fbd.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 include/linux/sched.h | 16 ++++++++++++++++
 init/Kconfig          |  3 +++
 2 files changed, 19 insertions(+)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index a287e8b..a958672 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -3094,11 +3094,19 @@ static inline struct thread_info *task_thread_info(struct task_struct *task)
 {
 	return &task->thread_info;
 }
+
+/*
+ * When accessing the stack of a non-current task that might exit, use
+ * try_get_task_stack() instead.  task_stack_page will return a pointer
+ * that could get freed out from under you.
+ */
 static inline void *task_stack_page(const struct task_struct *task)
 {
 	return task->stack;
 }
+
 #define setup_thread_stack(new,old)	do { } while(0)
+
 static inline unsigned long *end_of_stack(const struct task_struct *task)
 {
 	return task->stack;
@@ -3134,6 +3142,14 @@ static inline unsigned long *end_of_stack(struct task_struct *p)
 }
 
 #endif
+
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+	return task_stack_page(tsk);
+}
+
+static inline void put_task_stack(struct task_struct *tsk) {}
+
 #define task_stack_end_corrupted(task) \
 		(*(end_of_stack(task)) != STACK_END_MAGIC)
 
diff --git a/init/Kconfig b/init/Kconfig
index ec8d438..3b9a47f 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -33,6 +33,9 @@ config THREAD_INFO_IN_TASK
 	  make this work, an arch will need to remove all thread_info fields
 	  except flags and fix any runtime bugs.
 
+	  One subtle change that will be needed is to use try_get_task_stack()
+	  and put_task_stack() in save_thread_stack_tsk() and get_wchan().
+
 menu "General setup"
 
 config BROKEN

[PATCH v2 3/8] kthread: to_live_kthread() needs try_get_task_stack()

[Andy Lutomirski]

From: Oleg Nesterov <oleg@redhat.com>

get_task_struct(tsk) no longer pins tsk->stack so all users of
to_live_kthread() should do try_get_task_stack/put_task_stack to protect
"struct kthread" which lives on kthread's stack.

TODO: Kill to_live_kthread(), perhaps we can even kill "struct kthread" too,
and rework kthread_stop(), it can use task_work_add() to sync with the exiting
kernel thread.

Message-Id: <20160629180357.GA7178@redhat.com>
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 kernel/kthread.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/kernel/kthread.c b/kernel/kthread.c
index 9ff173dca1ae..4ab4c3766a80 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -64,7 +64,7 @@ static inline struct kthread *to_kthread(struct task_struct *k)
 static struct kthread *to_live_kthread(struct task_struct *k)
 {
 	struct completion *vfork = ACCESS_ONCE(k->vfork_done);
-	if (likely(vfork))
+	if (likely(vfork) && try_get_task_stack(k))
 		return __to_kthread(vfork);
 	return NULL;
 }
@@ -425,8 +425,10 @@ void kthread_unpark(struct task_struct *k)
 {
 	struct kthread *kthread = to_live_kthread(k);
 
-	if (kthread)
+	if (kthread) {
 		__kthread_unpark(k, kthread);
+		put_task_stack(k);
+	}
 }
 EXPORT_SYMBOL_GPL(kthread_unpark);
 
@@ -455,6 +457,7 @@ int kthread_park(struct task_struct *k)
 				wait_for_completion(&kthread->parked);
 			}
 		}
+		put_task_stack(k);
 		ret = 0;
 	}
 	return ret;
@@ -490,6 +493,7 @@ int kthread_stop(struct task_struct *k)
 		__kthread_unpark(k, kthread);
 		wake_up_process(k);
 		wait_for_completion(&kthread->exited);
+		put_task_stack(k);
 	}
 	ret = k->exit_code;
 	put_task_struct(k);
-- 
2.7.4

[tip:x86/asm] kthread: Pin the stack via try_get_task_stack()/put_task_stack() in to_live_kthread() function

[tip-bot for Oleg Nesterov]

Commit-ID:  23196f2e5f5d810578a772785807dcdc2b9fdce9
Gitweb:     http://git.kernel.org/tip/23196f2e5f5d810578a772785807dcdc2b9fdce9
Author:     Oleg Nesterov <oleg@redhat.com>
AuthorDate: Thu, 15 Sep 2016 22:45:44 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:53 +0200

kthread: Pin the stack via try_get_task_stack()/put_task_stack() in to_live_kthread() function

get_task_struct(tsk) no longer pins tsk->stack so all users of
to_live_kthread() should do try_get_task_stack/put_task_stack to protect
"struct kthread" which lives on kthread's stack.

TODO: Kill to_live_kthread(), perhaps we can even kill "struct kthread" too,
and rework kthread_stop(), it can use task_work_add() to sync with the exiting
kernel thread.

Message-Id: <20160629180357.GA7178@redhat.com>
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/cb9b16bbc19d4aea4507ab0552e4644c1211d130.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 kernel/kthread.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/kernel/kthread.c b/kernel/kthread.c
index 9ff173d..4ab4c37 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -64,7 +64,7 @@ static inline struct kthread *to_kthread(struct task_struct *k)
 static struct kthread *to_live_kthread(struct task_struct *k)
 {
 	struct completion *vfork = ACCESS_ONCE(k->vfork_done);
-	if (likely(vfork))
+	if (likely(vfork) && try_get_task_stack(k))
 		return __to_kthread(vfork);
 	return NULL;
 }
@@ -425,8 +425,10 @@ void kthread_unpark(struct task_struct *k)
 {
 	struct kthread *kthread = to_live_kthread(k);
 
-	if (kthread)
+	if (kthread) {
 		__kthread_unpark(k, kthread);
+		put_task_stack(k);
+	}
 }
 EXPORT_SYMBOL_GPL(kthread_unpark);
 
@@ -455,6 +457,7 @@ int kthread_park(struct task_struct *k)
 				wait_for_completion(&kthread->parked);
 			}
 		}
+		put_task_stack(k);
 		ret = 0;
 	}
 	return ret;
@@ -490,6 +493,7 @@ int kthread_stop(struct task_struct *k)
 		__kthread_unpark(k, kthread);
 		wake_up_process(k);
 		wait_for_completion(&kthread->exited);
+		put_task_stack(k);
 	}
 	ret = k->exit_code;
 	put_task_struct(k);

[PATCH v2 4/8] x86/dumpstack: Pin the target stack when dumping it

[Andy Lutomirski]

Specifically, pin the stack in save_stack_trace_tsk() and
show_trace_log_lvl().

This will prevent a crash if the target task dies before or while
dumping its stack once we start freeing task stacks early.

Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 arch/x86/kernel/dumpstack_32.c | 5 +++++
 arch/x86/kernel/dumpstack_64.c | 5 +++++
 arch/x86/kernel/stacktrace.c   | 5 +++++
 3 files changed, 15 insertions(+)

diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index 2d65cfa5e0b4..122f37d7bb7e 100644
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -163,6 +163,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	unsigned long *stack;
 	int i;
 
+	if (!try_get_task_stack(task))
+		return;
+
 	sp = sp ? : get_stack_pointer(task, regs);
 
 	stack = sp;
@@ -179,6 +182,8 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	}
 	pr_cont("\n");
 	show_trace_log_lvl(task, regs, sp, bp, log_lvl);
+
+	put_task_stack(task);
 }
 
 
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 8cb6004a4dfd..16c0d5f89b5e 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -218,6 +218,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	unsigned long *stack;
 	int i;
 
+	if (!try_get_task_stack(task))
+		return;
+
 	irq_stack_end = (unsigned long *)this_cpu_read(irq_stack_ptr);
 	irq_stack     = irq_stack_end - (IRQ_STACK_SIZE / sizeof(long));
 
@@ -253,6 +256,8 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 
 	pr_cont("\n");
 	show_trace_log_lvl(task, regs, sp, bp, log_lvl);
+
+	put_task_stack(task);
 }
 
 void show_regs(struct pt_regs *regs)
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index 785aef1c7ef5..23fa81e24c8a 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -79,9 +79,14 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
 
 void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
 {
+	if (!try_get_task_stack(tsk))
+		return;
+
 	dump_trace(tsk, NULL, NULL, 0, &save_stack_ops_nosched, trace);
 	if (trace->nr_entries < trace->max_entries)
 		trace->entries[trace->nr_entries++] = ULONG_MAX;
+
+	put_task_stack(tsk);
 }
 EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
 
-- 
2.7.4

[tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it

[tip-bot for Andy Lutomirski]

Commit-ID:  1959a60182f48879635812a03a99c02231ea8677
Gitweb:     http://git.kernel.org/tip/1959a60182f48879635812a03a99c02231ea8677
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:45 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:53 +0200

x86/dumpstack: Pin the target stack when dumping it

Specifically, pin the stack in save_stack_trace_tsk() and
show_trace_log_lvl().

This will prevent a crash if the target task dies before or while
dumping its stack once we start freeing task stacks early.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/cf0082cde65d1941a996d026f2b2cdbfaca17bfa.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/dumpstack_32.c | 5 +++++
 arch/x86/kernel/dumpstack_64.c | 5 +++++
 arch/x86/kernel/stacktrace.c   | 5 +++++
 3 files changed, 15 insertions(+)

diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index 2d65cfa..122f37d7 100644
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -163,6 +163,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	unsigned long *stack;
 	int i;
 
+	if (!try_get_task_stack(task))
+		return;
+
 	sp = sp ? : get_stack_pointer(task, regs);
 
 	stack = sp;
@@ -179,6 +182,8 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	}
 	pr_cont("\n");
 	show_trace_log_lvl(task, regs, sp, bp, log_lvl);
+
+	put_task_stack(task);
 }
 
 
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 8cb6004..16c0d5f 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -218,6 +218,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	unsigned long *stack;
 	int i;
 
+	if (!try_get_task_stack(task))
+		return;
+
 	irq_stack_end = (unsigned long *)this_cpu_read(irq_stack_ptr);
 	irq_stack     = irq_stack_end - (IRQ_STACK_SIZE / sizeof(long));
 
@@ -253,6 +256,8 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
 
 	pr_cont("\n");
 	show_trace_log_lvl(task, regs, sp, bp, log_lvl);
+
+	put_task_stack(task);
 }
 
 void show_regs(struct pt_regs *regs)
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index 785aef1..23fa81e 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -79,9 +79,14 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
 
 void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
 {
+	if (!try_get_task_stack(tsk))
+		return;
+
 	dump_trace(tsk, NULL, NULL, 0, &save_stack_ops_nosched, trace);
 	if (trace->nr_entries < trace->max_entries)
 		trace->entries[trace->nr_entries++] = ULONG_MAX;
+
+	put_task_stack(tsk);
 }
 EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
 

Re: [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it

[Josh Poimboeuf]

On Fri, Sep 16, 2016 at 02:17:46AM -0700, tip-bot for Andy Lutomirski wrote:
> Commit-ID:  1959a60182f48879635812a03a99c02231ea8677
> Gitweb:     http://git.kernel.org/tip/1959a60182f48879635812a03a99c02231ea8677
> Author:     Andy Lutomirski <luto@kernel.org>
> AuthorDate: Thu, 15 Sep 2016 22:45:45 -0700
> Committer:  Ingo Molnar <mingo@kernel.org>
> CommitDate: Fri, 16 Sep 2016 09:18:53 +0200
> 
> x86/dumpstack: Pin the target stack when dumping it
> 
> Specifically, pin the stack in save_stack_trace_tsk() and
> show_trace_log_lvl().
> 
> This will prevent a crash if the target task dies before or while
> dumping its stack once we start freeing task stacks early.

This causes a hang:

# echo 1 > /proc/sys/kernel/sysrq
# echo l > /proc/sysrq-trigger
[   26.553593] sysrq: SysRq : Show backtrace of all active CPUs
[   26.554130] Sending NMI to all CPUs:
[   26.554521] BAD LUCK: lost 2225 message(s) from NMI context!
[   26.554521] NMI backtrace for cpu 0
[   26.554521] CPU: 0 PID: 783 Comm: bash Not tainted 4.8.0-rc6+ #3
[   26.554522] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.1-1.fc24 04/01/2014
[   26.554522] task: ffff880074fbd480 task.stack: ffffc90000bcc000
[   26.554522] RIP: 0010:[<ffffffff81071526>]  [<ffffffff81071526>] native_write_msr+0x6/0x30
[   26.554523] RSP: 0018:ffffc90000bcfd18  EFLAGS: 00000046
[   26.554523] RAX: 0000000000000400 RBX: ffffffff81f83b20 RCX: 0000000000000830
[   26.554523] RDX: 0000000000000000 RSI: 0000000000000400 RDI: 0000000000000830
[   26.554523] RBP: ffffc90000bcfd20 R08: ffffffffffffffff R09: 0000000000000003
[   26.554524] R10: ffffffff81e5d1a0 R11: 0000000000000001 R12: 0000000000080000
[   26.554524] R13: 0000000000000000 R14: 000000000000a13c R15: 0000000000000000
[   26.554524] FS:  00007f0c7b8a2700(0000) GS:ffff88007d200000(0000) knlGS:0000000000000000
[   26.554525] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   26.554525] CR2: 00007f0c7b8a7000 CR3: 0000000074fff000 CR4: 00000000001406f0
[   26.554525] Stack:
[   26.554525] BUG: unable to handle kernel paging request at 00000000000025d0
[   26.554526] IP: [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554526] PGD 718d3067 PUD 718d2067 PMD 0 
[   26.554526] Oops: 0000 [#1] PREEMPT SMP
[   26.554527] Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack cfg80211 rfkill ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack snd_hda_codec_generic iptable_mangle iptable_security iptable_raw ppdev crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel snd_hda_intel snd_hda_codec serio_raw snd_hwdep snd_hda_core virtio_console virtio_balloon snd_pcm snd_timer snd virtio_net soundcore i2c_piix4 parport_pc parport virtio_blk ata_generic qxl drm_kms_helper ttm pata_acpi drm virtio_pci virtio_ring virtio
[   26.554528] CPU: 0 PID: 783 Comm: bash Not tainted 4.8.0-rc6+ #3
[   26.554528] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.1-1.fc24 04/01/2014
[   26.554528] task: ffff880074fbd480 task.stack: ffffc90000bcc000
[   26.554528] RIP: 0010:[<ffffffff81039ba8>]  [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554529] RSP: 0018:ffff88007d205d40  EFLAGS: 00010046
[   26.554529] RAX: 0000000000000000 RBX: ffff88007d205ef8 RCX: 0000000000000000
[   26.554529] RDX: 0000000000000000 RSI: ffff88007d205ef8 RDI: 0000000000000000
[   26.554530] RBP: ffff88007d205d98 R08: ffffffff81c35ef4 R09: 0000000000000005
[   26.554530] R10: ffff880074fbd480 R11: ffff88007d3ce637 R12: ffffffff81071501
[   26.554530] R13: ffff88007d205ef8 R14: 000000062ec39e9f R15: ffff88007d205ef8
[   26.554530] FS:  00007f0c7b8a2700(0000) GS:ffff88007d200000(0000) knlGS:0000000000000000
[   26.554531] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   26.554531] CR2: 00000000000025d0 CR3: 0000000074fff000 CR4: 00000000001406f0
[   26.554531] Stack:
[   26.554531] BUG: unable to handle kernel paging request at 00000000000025d0
[   26.554532] IP: [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554532] PGD 718d3067 PUD 718d2067 PMD 0 
[   26.554532] Oops: 0000 [#2] PREEMPT SMP
[   26.554533] Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack cfg80211 rfkill ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack snd_hda_codec_generic iptable_mangle iptable_security iptable_raw ppdev crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel snd_hda_intel snd_hda_codec serio_raw snd_hwdep snd_hda_core virtio_console virtio_balloon snd_pcm snd_timer snd virtio_net soundcore i2c_piix4 parport_pc parport virtio_blk ata_generic qxl drm_kms_helper ttm pata_acpi drm virtio_pci virtio_ring virtio
[   26.554533] CPU: 0 PID: 783 Comm: bash Not tainted 4.8.0-rc6+ #3
[   26.554534] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.1-1.fc24 04/01/2014
[   26.554534] task: ffff880074fbd480 task.stack: ffffc90000bcc000
[   26.554534] RIP: 0010:[<ffffffff81039ba8>]  [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554535] RSP: 0018:ffff88007d205a28  EFLAGS: 00010046
[   26.554535] RAX: 0000000000000000 RBX: ffff88007d205c98 RCX: 0000000000000000
[   26.554535] RDX: 0000000000000000 RSI: ffff88007d205c98 RDI: 0000000000000000
[   26.554535] RBP: ffff88007d205a80 R08: ffffffff81c35ef4 R09: 0000000000000005
[   26.554536] R10: ffff880074fbd480 R11: ffff88007d3ced4e R12: 0000000000000000
[   26.554536] R13: ffff88007d205c98 R14: 0000000000000000 R15: ffff88007d205c98
[   26.554536] FS:  00007f0c7b8a2700(0000) GS:ffff88007d200000(0000) knlGS:0000000000000000
[   26.554536] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   26.554537] CR2: 00000000000025d0 CR3: 0000000074fff000 CR4: 00000000001406f0
[   26.554537] Stack:
[   26.554537] BUG: unable to handle kernel paging request at 00000000000025d0
[   26.554537] IP: [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554538] PGD 718d3067 PUD 718d2067 PMD 0 
[   26.554538] Oops: 0000 [#3] PREEMPT SMP
[   26.554539] Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack cfg80211 rfkill ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack snd_hda_codec_generic iptable_mangle iptable_security iptable_raw ppdev crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel snd_hda_intel snd_hda_codec serio_raw snd_hwdep snd_hda_core virtio_console virtio_balloon snd_pcm snd_timer snd virtio_net soundcore i2c_piix4 parport_pc parport virtio_blk ata_generic qxl drm_kms_helper ttm pata_acpi drm virtio_pci virtio_ring virtio
[   26.554539] CPU: 0 PID: 783 Comm: bash Not tainted 4.8.0-rc6+ #3
[   26.554539] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.1-1.fc24 04/01/2014
[   26.554540] task: ffff880074fbd480 task.stack: ffffc90000bcc000
[   26.554540] RIP: 0010:[<ffffffff81039ba8>]  [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554540] RSP: 0018:ffff88007d205708  EFLAGS: 00010046
[   26.554540] RAX: 0000000000000000 RBX: ffff88007d205978 RCX: 0000000000000000
[   26.554541] RDX: 0000000000000000 RSI: ffff88007d205978 RDI: 0000000000000000
[   26.554541] RBP: ffff88007d205760 R08: ffffffff81c35ef4 R09: 0000000000000005
[   26.554541] R10: ffff880074fbd480 R11: ffff88007d3cf465 R12: 0000000000000000
[   26.554542] R13: ffff88007d205978 R14: 0000000000000000 R15: ffff88007d205978
[   26.554542] FS:  00007f0c7b8a2700(0000) GS:ffff88007d200000(0000) knlGS:0000000000000000
[   26.554542] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   26.554542] CR2: 00000000000025d0 CR3: 0000000074fff000 CR4: 00000000001406f0
[   26.554542] Stack:
[   26.554543] BUG: unable to handle kernel paging request at 00000000000025d0
[   26.554543] IP: [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554543] PGD 718d3067 PUD 718d2067 PMD 0 
[   26.554543] Oops: 0000 [#4] PREEMPT SMP
[   26.554545] Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack cfg80211 rfkill ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack snd_hda_codec_generic iptable_mangle iptable_security iptable_raw ppdev crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel snd_hda_intel snd_hda_codec serio_raw snd_hwdep snd_hda_core virtio_console virtio_balloon snd_pcm snd_timer snd virtio_net soundcore i2c_piix4 parport_pc parport virtio_blk ata_generic qxl drm_kms_helper ttm pata_acpi drm virtio_pci virtio_ring virtio
[   26.554545] CPU: 0 PID: 783 Comm: bash Not tainted 4.8.0-rc6+ #3
[   26.554545] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.1-1.fc24 04/01/2014
[   26.554545] task: ffff880074fbd480 task.stack: ffffc90000bcc000
[   26.554546] RIP: 0010:[<ffffffff81039ba8>]  [<ffffffff81039ba8>] show_stack_log_lvl+0x28/0x240
[   26.554546] RSP: 0018:ffff88007d2053e8  EFLAGS: 00010046
[   26.554546] RAX: 0000000000000000 RBX: ffff88007d205658 RCX: 0000000000000000
[   26.554546] RDX: 0000000000000000 RSI: ffff88007d205658 RDI: 0000000000000000
[   26.554547] RBP: ffff88007d205440 R08: ffffffff81c35ef4 R09: 0000000000000005
[   26.554547] R10: ffff880074fbd480 R11: ffff88007d3cfb7c R12: 0000000000000000
[   26.554547] R13: ffff88007d205658 R14: 0000000000000000 R15: ffff88007d205658
[   26.554547] FS:  00007f0c7b8a2700(0000) GS:ffff88007d200000(0000) knlGS:0000000000000000
[   26.554548] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   26.554548] CR2: 00000000000025d0 CR3: 0000000074fff000 CR4: 00000000001406f0
[   26.554548] Stack:
[   26.554549] BUG:
[   26.554632] Kernel Offset: disabled


-- 
Josh

Re: [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it

[Josh Poimboeuf]

On Fri, Sep 16, 2016 at 06:55:57AM -0500, Josh Poimboeuf wrote:
> On Fri, Sep 16, 2016 at 02:17:46AM -0700, tip-bot for Andy Lutomirski wrote:
> > Commit-ID:  1959a60182f48879635812a03a99c02231ea8677
> > Gitweb:     http://git.kernel.org/tip/1959a60182f48879635812a03a99c02231ea8677
> > Author:     Andy Lutomirski <luto@kernel.org>
> > AuthorDate: Thu, 15 Sep 2016 22:45:45 -0700
> > Committer:  Ingo Molnar <mingo@kernel.org>
> > CommitDate: Fri, 16 Sep 2016 09:18:53 +0200
> > 
> > x86/dumpstack: Pin the target stack when dumping it
> > 
> > Specifically, pin the stack in save_stack_trace_tsk() and
> > show_trace_log_lvl().
> > 
> > This will prevent a crash if the target task dies before or while
> > dumping its stack once we start freeing task stacks early.
> 
> This causes a hang:

The problem is that show_stack_log_lvl() can be called with a NULL
task_struct pointer to indicate 'current'.

No idea why that convention exists -- IMO we should just require the
caller to pass 'current' directly.

-- 
Josh

Re: [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it

[Ingo Molnar]

* Josh Poimboeuf <jpoimboe@redhat.com> wrote:

> On Fri, Sep 16, 2016 at 06:55:57AM -0500, Josh Poimboeuf wrote:
> > On Fri, Sep 16, 2016 at 02:17:46AM -0700, tip-bot for Andy Lutomirski wrote:
> > > Commit-ID:  1959a60182f48879635812a03a99c02231ea8677
> > > Gitweb:     http://git.kernel.org/tip/1959a60182f48879635812a03a99c02231ea8677
> > > Author:     Andy Lutomirski <luto@kernel.org>
> > > AuthorDate: Thu, 15 Sep 2016 22:45:45 -0700
> > > Committer:  Ingo Molnar <mingo@kernel.org>
> > > CommitDate: Fri, 16 Sep 2016 09:18:53 +0200
> > > 
> > > x86/dumpstack: Pin the target stack when dumping it
> > > 
> > > Specifically, pin the stack in save_stack_trace_tsk() and
> > > show_trace_log_lvl().
> > > 
> > > This will prevent a crash if the target task dies before or while
> > > dumping its stack once we start freeing task stacks early.
> > 
> > This causes a hang:
> 
> The problem is that show_stack_log_lvl() can be called with a NULL
> task_struct pointer to indicate 'current'.

Could you please send a quick fix that addresses the hang by turning the NULL into 
'current' or so?

> No idea why that convention exists -- IMO we should just require the
> caller to pass 'current' directly.

For hysterical raisins I believe. It appears just a single caller passes 
non-current. Feel free to change this.

Thanks,

	Ingo

[PATCH] x86/dumpstack: remove NULL task pointer convention

[Josh Poimboeuf]

On Fri, Sep 16, 2016 at 02:57:54PM +0200, Ingo Molnar wrote:
> 
> * Josh Poimboeuf <jpoimboe@redhat.com> wrote:
> 
> > On Fri, Sep 16, 2016 at 06:55:57AM -0500, Josh Poimboeuf wrote:
> > > On Fri, Sep 16, 2016 at 02:17:46AM -0700, tip-bot for Andy Lutomirski wrote:
> > > > Commit-ID:  1959a60182f48879635812a03a99c02231ea8677
> > > > Gitweb:     http://git.kernel.org/tip/1959a60182f48879635812a03a99c02231ea8677
> > > > Author:     Andy Lutomirski <luto@kernel.org>
> > > > AuthorDate: Thu, 15 Sep 2016 22:45:45 -0700
> > > > Committer:  Ingo Molnar <mingo@kernel.org>
> > > > CommitDate: Fri, 16 Sep 2016 09:18:53 +0200
> > > > 
> > > > x86/dumpstack: Pin the target stack when dumping it
> > > > 
> > > > Specifically, pin the stack in save_stack_trace_tsk() and
> > > > show_trace_log_lvl().
> > > > 
> > > > This will prevent a crash if the target task dies before or while
> > > > dumping its stack once we start freeing task stacks early.
> > > 
> > > This causes a hang:
> > 
> > The problem is that show_stack_log_lvl() can be called with a NULL
> > task_struct pointer to indicate 'current'.
> 
> Could you please send a quick fix that addresses the hang by turning the NULL into 
> 'current' or so?
> 
> > No idea why that convention exists -- IMO we should just require the
> > caller to pass 'current' directly.
> 
> For hysterical raisins I believe. It appears just a single caller passes 
> non-current. Feel free to change this.

I've verified that this fixes it.  It can be applied before Andy's patch
(or after if you don't want to rebase).

---

From: Josh Poimboeuf <jpoimboe@redhat.com>
Subject: [PATCH] x86/dumpstack: remove NULL task pointer convention

show_stack_log_lvl() and friends allow a NULL pointer for the
task_struct to indicate the current task.  This creates confusion and
can cause sneaky bugs.

Instead require the caller to pass 'current' directly.

This only changes the internal workings of the dumpstack code.  The
dump_trace() and show_stack() interfaces still allow a NULL task
pointer.  Those interfaces should also probably be fixed as well.

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
---
 arch/x86/include/asm/stacktrace.h | 4 ++--
 arch/x86/kernel/dumpstack.c       | 4 +++-
 arch/x86/kernel/dumpstack_32.c    | 2 +-
 arch/x86/kernel/dumpstack_64.c    | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h
index 780a83e..ed2be1b 100644
--- a/arch/x86/include/asm/stacktrace.h
+++ b/arch/x86/include/asm/stacktrace.h
@@ -94,7 +94,7 @@ get_frame_pointer(struct task_struct *task, struct pt_regs *regs)
 	if (regs)
 		return (unsigned long *)regs->bp;
 
-	if (!task || task == current)
+	if (task == current)
 		return __builtin_frame_address(0);
 
 	return (unsigned long *)((struct inactive_task_frame *)task->thread.sp)->bp;
@@ -113,7 +113,7 @@ get_stack_pointer(struct task_struct *task, struct pt_regs *regs)
 	if (regs)
 		return (unsigned long *)kernel_stack_pointer(regs);
 
-	if (!task || task == current)
+	if (task == current)
 		return __builtin_frame_address(0);
 
 	return (unsigned long *)task->thread.sp;
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index aa208e5..e0648f7 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -175,11 +175,13 @@ void show_stack(struct task_struct *task, unsigned long *sp)
 {
 	unsigned long bp = 0;
 
+	task = task ? : current;
+
 	/*
 	 * Stack frames below this one aren't interesting.  Don't show them
 	 * if we're printing for %current.
 	 */
-	if (!sp && (!task || task == current)) {
+	if (!sp && task == current) {
 		sp = get_stack_pointer(current, NULL);
 		bp = (unsigned long)get_frame_pointer(current, NULL);
 	}
diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index 2d65cfa..ded285c 100644
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -200,7 +200,7 @@ void show_regs(struct pt_regs *regs)
 		u8 *ip;
 
 		pr_emerg("Stack:\n");
-		show_stack_log_lvl(NULL, regs, NULL, 0, KERN_EMERG);
+		show_stack_log_lvl(current, regs, NULL, 0, KERN_EMERG);
 
 		pr_emerg("Code:");
 
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 8cb6004..4fac236 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -273,7 +273,7 @@ void show_regs(struct pt_regs *regs)
 		u8 *ip;
 
 		printk(KERN_DEFAULT "Stack:\n");
-		show_stack_log_lvl(NULL, regs, NULL, 0, KERN_DEFAULT);
+		show_stack_log_lvl(current, regs, NULL, 0, KERN_DEFAULT);
 
 		printk(KERN_DEFAULT "Code: ");
 
-- 
2.7.4

[PATCH v2 5/8] x86/process: Pin the target stack in get_wchan()

[Andy Lutomirski]

This will prevent a crash if get_wchan() runs after the task stack
is freed.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 arch/x86/kernel/process.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 0b9ed8ec5226..4002b475171c 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -532,15 +532,18 @@ unsigned long thread_saved_pc(struct task_struct *tsk)
  */
 unsigned long get_wchan(struct task_struct *p)
 {
-	unsigned long start, bottom, top, sp, fp, ip;
+	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
 	int count = 0;
 
 	if (!p || p == current || p->state == TASK_RUNNING)
 		return 0;
 
+	if (!try_get_task_stack(p))
+		return 0;
+
 	start = (unsigned long)task_stack_page(p);
 	if (!start)
-		return 0;
+		goto out;
 
 	/*
 	 * Layout of the stack page:
@@ -564,16 +567,21 @@ unsigned long get_wchan(struct task_struct *p)
 
 	sp = READ_ONCE(p->thread.sp);
 	if (sp < bottom || sp > top)
-		return 0;
+		goto out;
 
 	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
 	do {
 		if (fp < bottom || fp > top)
-			return 0;
+			goto out;
 		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
-		if (!in_sched_functions(ip))
-			return ip;
+		if (!in_sched_functions(ip)) {
+			ret = ip;
+			goto out;
+		}
 		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
 	} while (count++ < 16 && p->state != TASK_RUNNING);
-	return 0;
+
+out:
+	put_task_stack(p);
+	return ret;
 }
-- 
2.7.4

[tip:x86/asm] x86/process: Pin the target stack in get_wchan()

[tip-bot for Andy Lutomirski]

Commit-ID:  74327a3e884a0ff895ba7b51d3488e6a177407b2
Gitweb:     http://git.kernel.org/tip/74327a3e884a0ff895ba7b51d3488e6a177407b2
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:46 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:53 +0200

x86/process: Pin the target stack in get_wchan()

This will prevent a crash if get_wchan() runs after the task stack
is freed.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/337aeca8614024aa4d8d9c81053bbf8fcffbe4ad.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/process.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 0b9ed8e..4002b47 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -532,15 +532,18 @@ unsigned long thread_saved_pc(struct task_struct *tsk)
  */
 unsigned long get_wchan(struct task_struct *p)
 {
-	unsigned long start, bottom, top, sp, fp, ip;
+	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
 	int count = 0;
 
 	if (!p || p == current || p->state == TASK_RUNNING)
 		return 0;
 
+	if (!try_get_task_stack(p))
+		return 0;
+
 	start = (unsigned long)task_stack_page(p);
 	if (!start)
-		return 0;
+		goto out;
 
 	/*
 	 * Layout of the stack page:
@@ -564,16 +567,21 @@ unsigned long get_wchan(struct task_struct *p)
 
 	sp = READ_ONCE(p->thread.sp);
 	if (sp < bottom || sp > top)
-		return 0;
+		goto out;
 
 	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
 	do {
 		if (fp < bottom || fp > top)
-			return 0;
+			goto out;
 		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
-		if (!in_sched_functions(ip))
-			return ip;
+		if (!in_sched_functions(ip)) {
+			ret = ip;
+			goto out;
+		}
 		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
 	} while (count++ < 16 && p->state != TASK_RUNNING);
-	return 0;
+
+out:
+	put_task_stack(p);
+	return ret;
 }

[PATCH v2 6/8] lib/syscall: Pin the task stack in collect_syscall()

[Andy Lutomirski]

This will avoid a potential read-after-free if collect_syscall()
(e.g. /proc/PID/syscall) is called on an exiting task.

Reported-by: Jann Horn <jann@thejh.net>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 lib/syscall.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/syscall.c b/lib/syscall.c
index e30e03932480..63239e097b13 100644
--- a/lib/syscall.c
+++ b/lib/syscall.c
@@ -7,9 +7,19 @@ static int collect_syscall(struct task_struct *target, long *callno,
 			   unsigned long args[6], unsigned int maxargs,
 			   unsigned long *sp, unsigned long *pc)
 {
-	struct pt_regs *regs = task_pt_regs(target);
-	if (unlikely(!regs))
+	struct pt_regs *regs;
+
+	if (!try_get_task_stack(target)) {
+		/* Task has no stack, so the task isn't in a syscall. */
+		*callno = -1;
+		return 0;
+	}
+
+	regs = task_pt_regs(target);
+	if (unlikely(!regs)) {
+		put_task_stack(target);
 		return -EAGAIN;
+	}
 
 	*sp = user_stack_pointer(regs);
 	*pc = instruction_pointer(regs);
@@ -18,6 +28,7 @@ static int collect_syscall(struct task_struct *target, long *callno,
 	if (*callno != -1L && maxargs > 0)
 		syscall_get_arguments(target, regs, 0, maxargs, args);
 
+	put_task_stack(target);
 	return 0;
 }
 
-- 
2.7.4

[tip:x86/asm] lib/syscall: Pin the task stack in collect_syscall()

[tip-bot for Andy Lutomirski]

Commit-ID:  aa1f1a639621672b68f654dc815a7d8298ff396f
Gitweb:     http://git.kernel.org/tip/aa1f1a639621672b68f654dc815a7d8298ff396f
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:47 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:53 +0200

lib/syscall: Pin the task stack in collect_syscall()

This will avoid a potential read-after-free if collect_syscall()
(e.g. /proc/PID/syscall) is called on an exiting task.

Reported-by: Jann Horn <jann@thejh.net>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/0bfd8e6d4729c97745d3781a29610a33d0a8091d.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 lib/syscall.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/syscall.c b/lib/syscall.c
index e30e039..63239e0 100644
--- a/lib/syscall.c
+++ b/lib/syscall.c
@@ -7,9 +7,19 @@ static int collect_syscall(struct task_struct *target, long *callno,
 			   unsigned long args[6], unsigned int maxargs,
 			   unsigned long *sp, unsigned long *pc)
 {
-	struct pt_regs *regs = task_pt_regs(target);
-	if (unlikely(!regs))
+	struct pt_regs *regs;
+
+	if (!try_get_task_stack(target)) {
+		/* Task has no stack, so the task isn't in a syscall. */
+		*callno = -1;
+		return 0;
+	}
+
+	regs = task_pt_regs(target);
+	if (unlikely(!regs)) {
+		put_task_stack(target);
 		return -EAGAIN;
+	}
 
 	*sp = user_stack_pointer(regs);
 	*pc = instruction_pointer(regs);
@@ -18,6 +28,7 @@ static int collect_syscall(struct task_struct *target, long *callno,
 	if (*callno != -1L && maxargs > 0)
 		syscall_get_arguments(target, regs, 0, maxargs, args);
 
+	put_task_stack(target);
 	return 0;
 }
 

[PATCH v2 7/8] sched: Free the stack early if CONFIG_THREAD_INFO_IN_TASK

[Andy Lutomirski]

We currently keep every task's stack around until the task_struct
itself is freed.  This means that we keep the stack allocation alive
for longer than necessary and that, under load, we free stacks in
big batches whenever RCU drops the last task reference.  Neither of
these is good for reuse of cache-hot memory, and freeing in batches
prevents us from usefully caching small numbers of vmalloced stacks.

On architectures that have thread_info on the stack, we can't easily
change this, but on architectures that set THREAD_INFO_IN_TASK, we
can free it as soon as the task is dead.

Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 include/linux/init_task.h |  4 +++-
 include/linux/sched.h     | 14 ++++++++++++++
 kernel/fork.c             | 35 ++++++++++++++++++++++++++++++++++-
 kernel/sched/core.c       |  4 ++++
 4 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 9c04d44eeb3c..325f649d77ff 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -186,7 +186,9 @@ extern struct task_group root_task_group;
 #endif
 
 #ifdef CONFIG_THREAD_INFO_IN_TASK
-# define INIT_TASK_TI(tsk) .thread_info = INIT_THREAD_INFO(tsk),
+# define INIT_TASK_TI(tsk)			\
+	.thread_info = INIT_THREAD_INFO(tsk),	\
+	.stack_refcount = ATOMIC_INIT(1),
 #else
 # define INIT_TASK_TI(tsk)
 #endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a95867267e9f..abb795afc823 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1936,6 +1936,10 @@ struct task_struct {
 #ifdef CONFIG_VMAP_STACK
 	struct vm_struct *stack_vm_area;
 #endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+	/* A live task holds one reference. */
+	atomic_t stack_refcount;
+#endif
 /* CPU-specific state of this task */
 	struct thread_struct thread;
 /*
@@ -3143,12 +3147,22 @@ static inline unsigned long *end_of_stack(struct task_struct *p)
 
 #endif
 
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+	return atomic_inc_not_zero(&tsk->stack_refcount) ?
+		task_stack_page(tsk) : NULL;
+}
+
+extern void put_task_stack(struct task_struct *tsk);
+#else
 static inline void *try_get_task_stack(struct task_struct *tsk)
 {
 	return task_stack_page(tsk);
 }
 
 static inline void put_task_stack(struct task_struct *tsk) {}
+#endif
 
 #define task_stack_end_corrupted(task) \
 		(*(end_of_stack(task)) != STACK_END_MAGIC)
diff --git a/kernel/fork.c b/kernel/fork.c
index 0c240fd5beba..5dd0a516626d 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -269,11 +269,40 @@ static void account_kernel_stack(struct task_struct *tsk, int account)
 	}
 }
 
-void free_task(struct task_struct *tsk)
+static void release_task_stack(struct task_struct *tsk)
 {
 	account_kernel_stack(tsk, -1);
 	arch_release_thread_stack(tsk->stack);
 	free_thread_stack(tsk);
+	tsk->stack = NULL;
+#ifdef CONFIG_VMAP_STACK
+	tsk->stack_vm_area = NULL;
+#endif
+}
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+void put_task_stack(struct task_struct *tsk)
+{
+	if (atomic_dec_and_test(&tsk->stack_refcount))
+		release_task_stack(tsk);
+}
+#endif
+
+void free_task(struct task_struct *tsk)
+{
+#ifndef CONFIG_THREAD_INFO_IN_TASK
+	/*
+	 * The task is finally done with both the stack and thread_info,
+	 * so free both.
+	 */
+	release_task_stack(tsk);
+#else
+	/*
+	 * If the task had a separate stack allocation, it should be gone
+	 * by now.
+	 */
+	WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0);
+#endif
 	rt_mutex_debug_task_free(tsk);
 	ftrace_graph_exit_task(tsk);
 	put_seccomp_filter(tsk);
@@ -411,6 +440,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 #ifdef CONFIG_VMAP_STACK
 	tsk->stack_vm_area = stack_vm_area;
 #endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+	atomic_set(&tsk->stack_refcount, 1);
+#endif
 
 	if (err)
 		goto free_stack;
@@ -1771,6 +1803,7 @@ bad_fork_cleanup_count:
 	atomic_dec(&p->cred->user->processes);
 	exit_creds(p);
 bad_fork_free:
+	put_task_stack(p);
 	free_task(p);
 fork_out:
 	return ERR_PTR(retval);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0b6238f18da2..23c6037e2d89 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2772,6 +2772,10 @@ static struct rq *finish_task_switch(struct task_struct *prev)
 		 * task and put them back on the free list.
 		 */
 		kprobe_flush_task(prev);
+
+		/* Task is done with its stack. */
+		put_task_stack(prev);
+
 		put_task_struct(prev);
 	}
 
-- 
2.7.4

[tip:x86/asm] sched/core: Free the stack early if CONFIG_THREAD_INFO_IN_TASK

[tip-bot for Andy Lutomirski]

Commit-ID:  68f24b08ee892d47bdef925d676e1ae1ccc316f8
Gitweb:     http://git.kernel.org/tip/68f24b08ee892d47bdef925d676e1ae1ccc316f8
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:48 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:54 +0200

sched/core: Free the stack early if CONFIG_THREAD_INFO_IN_TASK

We currently keep every task's stack around until the task_struct
itself is freed.  This means that we keep the stack allocation alive
for longer than necessary and that, under load, we free stacks in
big batches whenever RCU drops the last task reference.  Neither of
these is good for reuse of cache-hot memory, and freeing in batches
prevents us from usefully caching small numbers of vmalloced stacks.

On architectures that have thread_info on the stack, we can't easily
change this, but on architectures that set THREAD_INFO_IN_TASK, we
can free it as soon as the task is dead.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/08ca06cde00ebed0046c5d26cbbf3fbb7ef5b812.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 include/linux/init_task.h |  4 +++-
 include/linux/sched.h     | 14 ++++++++++++++
 kernel/fork.c             | 35 ++++++++++++++++++++++++++++++++++-
 kernel/sched/core.c       |  4 ++++
 4 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 9c04d44..325f649 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -186,7 +186,9 @@ extern struct task_group root_task_group;
 #endif
 
 #ifdef CONFIG_THREAD_INFO_IN_TASK
-# define INIT_TASK_TI(tsk) .thread_info = INIT_THREAD_INFO(tsk),
+# define INIT_TASK_TI(tsk)			\
+	.thread_info = INIT_THREAD_INFO(tsk),	\
+	.stack_refcount = ATOMIC_INIT(1),
 #else
 # define INIT_TASK_TI(tsk)
 #endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a958672..abb795a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1936,6 +1936,10 @@ struct task_struct {
 #ifdef CONFIG_VMAP_STACK
 	struct vm_struct *stack_vm_area;
 #endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+	/* A live task holds one reference. */
+	atomic_t stack_refcount;
+#endif
 /* CPU-specific state of this task */
 	struct thread_struct thread;
 /*
@@ -3143,12 +3147,22 @@ static inline unsigned long *end_of_stack(struct task_struct *p)
 
 #endif
 
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+	return atomic_inc_not_zero(&tsk->stack_refcount) ?
+		task_stack_page(tsk) : NULL;
+}
+
+extern void put_task_stack(struct task_struct *tsk);
+#else
 static inline void *try_get_task_stack(struct task_struct *tsk)
 {
 	return task_stack_page(tsk);
 }
 
 static inline void put_task_stack(struct task_struct *tsk) {}
+#endif
 
 #define task_stack_end_corrupted(task) \
 		(*(end_of_stack(task)) != STACK_END_MAGIC)
diff --git a/kernel/fork.c b/kernel/fork.c
index 0c240fd..5dd0a51 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -269,11 +269,40 @@ static void account_kernel_stack(struct task_struct *tsk, int account)
 	}
 }
 
-void free_task(struct task_struct *tsk)
+static void release_task_stack(struct task_struct *tsk)
 {
 	account_kernel_stack(tsk, -1);
 	arch_release_thread_stack(tsk->stack);
 	free_thread_stack(tsk);
+	tsk->stack = NULL;
+#ifdef CONFIG_VMAP_STACK
+	tsk->stack_vm_area = NULL;
+#endif
+}
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+void put_task_stack(struct task_struct *tsk)
+{
+	if (atomic_dec_and_test(&tsk->stack_refcount))
+		release_task_stack(tsk);
+}
+#endif
+
+void free_task(struct task_struct *tsk)
+{
+#ifndef CONFIG_THREAD_INFO_IN_TASK
+	/*
+	 * The task is finally done with both the stack and thread_info,
+	 * so free both.
+	 */
+	release_task_stack(tsk);
+#else
+	/*
+	 * If the task had a separate stack allocation, it should be gone
+	 * by now.
+	 */
+	WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0);
+#endif
 	rt_mutex_debug_task_free(tsk);
 	ftrace_graph_exit_task(tsk);
 	put_seccomp_filter(tsk);
@@ -411,6 +440,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 #ifdef CONFIG_VMAP_STACK
 	tsk->stack_vm_area = stack_vm_area;
 #endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+	atomic_set(&tsk->stack_refcount, 1);
+#endif
 
 	if (err)
 		goto free_stack;
@@ -1771,6 +1803,7 @@ bad_fork_cleanup_count:
 	atomic_dec(&p->cred->user->processes);
 	exit_creds(p);
 bad_fork_free:
+	put_task_stack(p);
 	free_task(p);
 fork_out:
 	return ERR_PTR(retval);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 0b6238f..23c6037 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2772,6 +2772,10 @@ static struct rq *finish_task_switch(struct task_struct *prev)
 		 * task and put them back on the free list.
 		 */
 		kprobe_flush_task(prev);
+
+		/* Task is done with its stack. */
+		put_task_stack(prev);
+
 		put_task_struct(prev);
 	}
 

[PATCH v2 8/8] fork: Cache two thread stacks per cpu if CONFIG_VMAP_STACK is set

[Andy Lutomirski]

vmalloc is a bit slow, and pounding vmalloc/vfree will eventually
force a global TLB flush.

To reduce pressure on them, if CONFIG_VMAP_STACK, cache two thread
stacks per cpu.  This will let us quickly allocate a hopefully
cache-hot, TLB-hot stack under heavy forking workloads (shell script
style).

On my silly pthread_create benchmark, it saves about 2 µs per
pthread_create+join with CONFIG_VMAP_STACK=y.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 kernel/fork.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/kernel/fork.c b/kernel/fork.c
index 5dd0a516626d..2d44a9d05218 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -159,15 +159,41 @@ void __weak arch_release_thread_stack(unsigned long *stack)
  * kmemcache based allocator.
  */
 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
+
+#ifdef CONFIG_VMAP_STACK
+/*
+ * vmalloc is a bit slow, and calling vfree enough times will force a TLB
+ * flush.  Try to minimize the number of calls by caching stacks.
+ */
+#define NR_CACHED_STACKS 2
+static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
+#endif
+
 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 {
 #ifdef CONFIG_VMAP_STACK
-	void *stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
-					   VMALLOC_START, VMALLOC_END,
-					   THREADINFO_GFP | __GFP_HIGHMEM,
-					   PAGE_KERNEL,
-					   0, node,
-					   __builtin_return_address(0));
+	void *stack;
+	int i;
+
+	local_irq_disable();
+	for (i = 0; i < NR_CACHED_STACKS; i++) {
+		struct vm_struct *s = this_cpu_read(cached_stacks[i]);
+
+		if (!s)
+			continue;
+		this_cpu_write(cached_stacks[i], NULL);
+
+		tsk->stack_vm_area = s;
+		local_irq_enable();
+		return s->addr;
+	}
+	local_irq_enable();
+
+	stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
+				     VMALLOC_START, VMALLOC_END,
+				     THREADINFO_GFP | __GFP_HIGHMEM,
+				     PAGE_KERNEL,
+				     0, node, __builtin_return_address(0));
 
 	/*
 	 * We can't call find_vm_area() in interrupt context, and
@@ -187,10 +213,28 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 
 static inline void free_thread_stack(struct task_struct *tsk)
 {
-	if (task_stack_vm_area(tsk))
+#ifdef CONFIG_VMAP_STACK
+	if (task_stack_vm_area(tsk)) {
+		unsigned long flags;
+		int i;
+
+		local_irq_save(flags);
+		for (i = 0; i < NR_CACHED_STACKS; i++) {
+			if (this_cpu_read(cached_stacks[i]))
+				continue;
+
+			this_cpu_write(cached_stacks[i], tsk->stack_vm_area);
+			local_irq_restore(flags);
+			return;
+		}
+		local_irq_restore(flags);
+
 		vfree(tsk->stack);
-	else
-		__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
+		return;
+	}
+#endif
+
+	__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
 }
 # else
 static struct kmem_cache *thread_stack_cache;
-- 
2.7.4

[tip:x86/asm] fork: Optimize task creation by caching two thread stacks per CPU if CONFIG_VMAP_STACK=y

[tip-bot for Andy Lutomirski]

Commit-ID:  ac496bf48d97f2503eaa353996a4dd5e4383eaf0
Gitweb:     http://git.kernel.org/tip/ac496bf48d97f2503eaa353996a4dd5e4383eaf0
Author:     Andy Lutomirski <luto@kernel.org>
AuthorDate: Thu, 15 Sep 2016 22:45:49 -0700
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 16 Sep 2016 09:18:54 +0200

fork: Optimize task creation by caching two thread stacks per CPU if CONFIG_VMAP_STACK=y

vmalloc() is a bit slow, and pounding vmalloc()/vfree() will eventually
force a global TLB flush.

To reduce pressure on them, if CONFIG_VMAP_STACK=y, cache two thread
stacks per CPU.  This will let us quickly allocate a hopefully
cache-hot, TLB-hot stack under heavy forking workloads (shell script style).

On my silly pthread_create() benchmark, it saves about 2 µs per
pthread_create()+join() with CONFIG_VMAP_STACK=y.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jann Horn <jann@thejh.net>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/94811d8e3994b2e962f88866290017d498eb069c.1474003868.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 kernel/fork.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/kernel/fork.c b/kernel/fork.c
index 5dd0a51..c060c7e 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -159,15 +159,41 @@ void __weak arch_release_thread_stack(unsigned long *stack)
  * kmemcache based allocator.
  */
 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
+
+#ifdef CONFIG_VMAP_STACK
+/*
+ * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
+ * flush.  Try to minimize the number of calls by caching stacks.
+ */
+#define NR_CACHED_STACKS 2
+static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
+#endif
+
 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 {
 #ifdef CONFIG_VMAP_STACK
-	void *stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
-					   VMALLOC_START, VMALLOC_END,
-					   THREADINFO_GFP | __GFP_HIGHMEM,
-					   PAGE_KERNEL,
-					   0, node,
-					   __builtin_return_address(0));
+	void *stack;
+	int i;
+
+	local_irq_disable();
+	for (i = 0; i < NR_CACHED_STACKS; i++) {
+		struct vm_struct *s = this_cpu_read(cached_stacks[i]);
+
+		if (!s)
+			continue;
+		this_cpu_write(cached_stacks[i], NULL);
+
+		tsk->stack_vm_area = s;
+		local_irq_enable();
+		return s->addr;
+	}
+	local_irq_enable();
+
+	stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
+				     VMALLOC_START, VMALLOC_END,
+				     THREADINFO_GFP | __GFP_HIGHMEM,
+				     PAGE_KERNEL,
+				     0, node, __builtin_return_address(0));
 
 	/*
 	 * We can't call find_vm_area() in interrupt context, and
@@ -187,10 +213,28 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 
 static inline void free_thread_stack(struct task_struct *tsk)
 {
-	if (task_stack_vm_area(tsk))
+#ifdef CONFIG_VMAP_STACK
+	if (task_stack_vm_area(tsk)) {
+		unsigned long flags;
+		int i;
+
+		local_irq_save(flags);
+		for (i = 0; i < NR_CACHED_STACKS; i++) {
+			if (this_cpu_read(cached_stacks[i]))
+				continue;
+
+			this_cpu_write(cached_stacks[i], tsk->stack_vm_area);
+			local_irq_restore(flags);
+			return;
+		}
+		local_irq_restore(flags);
+
 		vfree(tsk->stack);
-	else
-		__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
+		return;
+	}
+#endif
+
+	__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
 }
 # else
 static struct kmem_cache *thread_stack_cache;