From: Hoeun Ryu <hoeun.ryu@gmail.com> To: Andrew Morton <akpm@linux-foundation.org>, Michal Hocko <mhocko@suse.com>, Ingo Molnar <mingo@kernel.org>, Andy Lutomirski <luto@kernel.org>, Kees Cook <keescook@chromium.org>, "Eric W. Biederman" <ebiederm@xmission.com>, Mateusz Guzik <mguzik@redhat.com> Cc: linux-kernel@vger.kernel.org, kernel-hardening@lists.openwall.com, Hoeun Ryu <hoeun.ryu@gmail.com> Subject: [PATCH 1/3] fork: dynamically allocate cache array for vmapped stacks using cpuhp Date: Sat, 4 Feb 2017 00:30:05 +0900 [thread overview] Message-ID: <1486135892-27249-1-git-send-email-hoeun.ryu@gmail.com> (raw) Using virtually mapped stack, kernel stacks are allocated via vmalloc. In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api. In this new implementation, the array for the cached stacks are dynamically allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established in fork_init(). Signed-off-by: Hoeun Ryu <hoeun.ryu@gmail.com> --- kernel/fork.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 61284d8..54421a9 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -167,26 +167,71 @@ void __weak arch_release_thread_stack(unsigned long *stack) * flush. Try to minimize the number of calls by caching stacks. */ #define NR_CACHED_STACKS 2 -static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +struct vm_stack_cache { + struct vm_struct **vm_stacks; + int nr; + int cur; +}; + +static DEFINE_PER_CPU(struct vm_stack_cache, vm_stacks); + +static int alloc_vm_stack_cache(unsigned int cpu) +{ + struct vm_stack_cache *vm_stack_cache = &per_cpu(vm_stacks, cpu); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; + int i; + + /* if free_vm_stack_cache() didn't free it */ + if (!vm_stacks) { + vm_stacks = + vzalloc(sizeof(struct vm_struct *) * NR_CACHED_STACKS); + if (!vm_stacks) + return -ENOMEM; + } + + vm_stack_cache->vm_stacks = vm_stacks; + vm_stack_cache->cur = 0; + vm_stack_cache->nr = 0; + + return 0; +} + +static int free_vm_stack_cache(unsigned int cpu) +{ + struct vm_stack_cache *vm_stack_cache = &per_cpu(vm_stacks, cpu); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; + int i; + + for (i = 0; i < vm_stack_cache->nr; i++) { + vfree(vm_stacks[i]->addr); + vm_stacks[i] = NULL; + } + + vm_stack_cache->nr = 0; + vm_stack_cache->cur = 0; + /* do not free vm_stack[cpu]->vm_stacks itself, reused in allocation */ + + return 0; +} + #endif static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) { #ifdef CONFIG_VMAP_STACK + struct vm_stack_cache *vm_stack_cache = + &per_cpu(vm_stacks, smp_processor_id()); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; void *stack; - int i; local_irq_disable(); - for (i = 0; i < NR_CACHED_STACKS; i++) { - struct vm_struct *s = this_cpu_read(cached_stacks[i]); - - if (!s) - continue; - this_cpu_write(cached_stacks[i], NULL); - - tsk->stack_vm_area = s; + if (vm_stack_cache->cur > 0) { + struct vm_struct *vm_stack = vm_stacks[--vm_stack_cache->cur]; + tsk->stack_vm_area = vm_stack; local_irq_enable(); - return s->addr; + + return vm_stack->addr; } local_irq_enable(); @@ -216,15 +261,14 @@ static inline void free_thread_stack(struct task_struct *tsk) { #ifdef CONFIG_VMAP_STACK if (task_stack_vm_area(tsk)) { + struct vm_stack_cache *vm_stack_cache = + &per_cpu(vm_stacks, smp_processor_id()); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; unsigned long flags; - int i; local_irq_save(flags); - for (i = 0; i < NR_CACHED_STACKS; i++) { - if (this_cpu_read(cached_stacks[i])) - continue; - - this_cpu_write(cached_stacks[i], tsk->stack_vm_area); + if (vm_stack_cache->cur < vm_stack_cache->nr) { + vm_stacks[vm_stack_cache->cur++] = tsk->stack_vm_area; local_irq_restore(flags); return; } @@ -456,6 +500,9 @@ void __init fork_init(void) for (i = 0; i < UCOUNT_COUNTS; i++) { init_user_ns.ucount_max[i] = max_threads/2; } + + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "vm_stack_cache", + alloc_vm_stack_cache, free_vm_stack_cache); } int __weak arch_dup_task_struct(struct task_struct *dst, -- 2.7.4
WARNING: multiple messages have this Message-ID (diff)
From: Hoeun Ryu <hoeun.ryu@gmail.com> To: Andrew Morton <akpm@linux-foundation.org>, Michal Hocko <mhocko@suse.com>, Ingo Molnar <mingo@kernel.org>, Andy Lutomirski <luto@kernel.org>, Kees Cook <keescook@chromium.org>, "Eric W. Biederman" <ebiederm@xmission.com>, Mateusz Guzik <mguzik@redhat.com> Cc: linux-kernel@vger.kernel.org, kernel-hardening@lists.openwall.com, Hoeun Ryu <hoeun.ryu@gmail.com> Subject: [kernel-hardening] [PATCH 1/3] fork: dynamically allocate cache array for vmapped stacks using cpuhp Date: Sat, 4 Feb 2017 00:30:05 +0900 [thread overview] Message-ID: <1486135892-27249-1-git-send-email-hoeun.ryu@gmail.com> (raw) Using virtually mapped stack, kernel stacks are allocated via vmalloc. In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api. In this new implementation, the array for the cached stacks are dynamically allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established in fork_init(). Signed-off-by: Hoeun Ryu <hoeun.ryu@gmail.com> --- kernel/fork.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 61284d8..54421a9 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -167,26 +167,71 @@ void __weak arch_release_thread_stack(unsigned long *stack) * flush. Try to minimize the number of calls by caching stacks. */ #define NR_CACHED_STACKS 2 -static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +struct vm_stack_cache { + struct vm_struct **vm_stacks; + int nr; + int cur; +}; + +static DEFINE_PER_CPU(struct vm_stack_cache, vm_stacks); + +static int alloc_vm_stack_cache(unsigned int cpu) +{ + struct vm_stack_cache *vm_stack_cache = &per_cpu(vm_stacks, cpu); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; + int i; + + /* if free_vm_stack_cache() didn't free it */ + if (!vm_stacks) { + vm_stacks = + vzalloc(sizeof(struct vm_struct *) * NR_CACHED_STACKS); + if (!vm_stacks) + return -ENOMEM; + } + + vm_stack_cache->vm_stacks = vm_stacks; + vm_stack_cache->cur = 0; + vm_stack_cache->nr = 0; + + return 0; +} + +static int free_vm_stack_cache(unsigned int cpu) +{ + struct vm_stack_cache *vm_stack_cache = &per_cpu(vm_stacks, cpu); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; + int i; + + for (i = 0; i < vm_stack_cache->nr; i++) { + vfree(vm_stacks[i]->addr); + vm_stacks[i] = NULL; + } + + vm_stack_cache->nr = 0; + vm_stack_cache->cur = 0; + /* do not free vm_stack[cpu]->vm_stacks itself, reused in allocation */ + + return 0; +} + #endif static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) { #ifdef CONFIG_VMAP_STACK + struct vm_stack_cache *vm_stack_cache = + &per_cpu(vm_stacks, smp_processor_id()); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; void *stack; - int i; local_irq_disable(); - for (i = 0; i < NR_CACHED_STACKS; i++) { - struct vm_struct *s = this_cpu_read(cached_stacks[i]); - - if (!s) - continue; - this_cpu_write(cached_stacks[i], NULL); - - tsk->stack_vm_area = s; + if (vm_stack_cache->cur > 0) { + struct vm_struct *vm_stack = vm_stacks[--vm_stack_cache->cur]; + tsk->stack_vm_area = vm_stack; local_irq_enable(); - return s->addr; + + return vm_stack->addr; } local_irq_enable(); @@ -216,15 +261,14 @@ static inline void free_thread_stack(struct task_struct *tsk) { #ifdef CONFIG_VMAP_STACK if (task_stack_vm_area(tsk)) { + struct vm_stack_cache *vm_stack_cache = + &per_cpu(vm_stacks, smp_processor_id()); + struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; unsigned long flags; - int i; local_irq_save(flags); - for (i = 0; i < NR_CACHED_STACKS; i++) { - if (this_cpu_read(cached_stacks[i])) - continue; - - this_cpu_write(cached_stacks[i], tsk->stack_vm_area); + if (vm_stack_cache->cur < vm_stack_cache->nr) { + vm_stacks[vm_stack_cache->cur++] = tsk->stack_vm_area; local_irq_restore(flags); return; } @@ -456,6 +500,9 @@ void __init fork_init(void) for (i = 0; i < UCOUNT_COUNTS; i++) { init_user_ns.ucount_max[i] = max_threads/2; } + + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "vm_stack_cache", + alloc_vm_stack_cache, free_vm_stack_cache); } int __weak arch_dup_task_struct(struct task_struct *dst, -- 2.7.4
next reply other threads:[~2017-02-03 15:38 UTC|newest] Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top 2017-02-03 15:30 Hoeun Ryu [this message] 2017-02-03 15:30 ` [kernel-hardening] [PATCH 1/3] fork: dynamically allocate cache array for vmapped stacks using cpuhp Hoeun Ryu 2017-02-03 15:39 ` Michal Hocko 2017-02-03 15:39 ` [kernel-hardening] " Michal Hocko 2017-02-03 16:42 ` Hoeun Ryu 2017-02-03 16:42 ` [kernel-hardening] " Hoeun Ryu 2017-02-03 17:15 ` Michal Hocko 2017-02-03 17:15 ` [kernel-hardening] " Michal Hocko 2017-02-03 17:52 ` Andy Lutomirski 2017-02-03 17:52 ` [kernel-hardening] " Andy Lutomirski 2017-02-04 2:01 ` Hoeun Ryu 2017-02-04 2:01 ` [kernel-hardening] " Hoeun Ryu 2017-02-05 10:18 ` Michal Hocko 2017-02-05 10:18 ` [kernel-hardening] " Michal Hocko 2017-02-05 13:23 ` Hoeun Ryu 2017-02-05 13:23 ` [kernel-hardening] " Hoeun Ryu 2017-02-03 16:04 ` kbuild test robot 2017-02-03 16:04 ` [kernel-hardening] " kbuild test robot
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=1486135892-27249-1-git-send-email-hoeun.ryu@gmail.com \ --to=hoeun.ryu@gmail.com \ --cc=akpm@linux-foundation.org \ --cc=ebiederm@xmission.com \ --cc=keescook@chromium.org \ --cc=kernel-hardening@lists.openwall.com \ --cc=linux-kernel@vger.kernel.org \ --cc=luto@kernel.org \ --cc=mguzik@redhat.com \ --cc=mhocko@suse.com \ --cc=mingo@kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.