From: Christoph Hellwig <hch@lst.de> To: akpm@linux-foundation.org Cc: joelaf@google.com, jszhang@marvell.com, chris@chris-wilson.co.uk, joaodias@google.com, linux-mm@kvack.org, linux-rt-users@vger.kernel.org, linux-kernel@vger.kernel.org Subject: [PATCH 1/6] mm: refactor __purge_vmap_area_lazy Date: Tue, 18 Oct 2016 08:56:06 +0200 [thread overview] Message-ID: <1476773771-11470-2-git-send-email-hch@lst.de> (raw) In-Reply-To: <1476773771-11470-1-git-send-email-hch@lst.de> Move the purge_lock synchronization to the callers, move the call to purge_fragmented_blocks_allcpus at the beginning of the function to the callers that need it, move the force_flush behavior to the caller that needs it, and pass start and end by value instead of by reference. No change in behavior. Signed-off-by: Christoph Hellwig <hch@lst.de> --- mm/vmalloc.c | 80 ++++++++++++++++++++++++++---------------------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f2481cb..d045a10 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -601,6 +601,13 @@ static unsigned long lazy_max_pages(void) static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); +/* + * Serialize vmap purging. There is no actual criticial section protected + * by this look, but we want to avoid concurrent calls for performance + * reasons and to make the pcpu_get_vm_areas more deterministic. + */ +static DEFINE_SPINLOCK(vmap_purge_lock); + /* for per-CPU blocks */ static void purge_fragmented_blocks_allcpus(void); @@ -615,59 +622,36 @@ void set_iounmap_nonlazy(void) /* * Purges all lazily-freed vmap areas. - * - * If sync is 0 then don't purge if there is already a purge in progress. - * If force_flush is 1, then flush kernel TLBs between *start and *end even - * if we found no lazy vmap areas to unmap (callers can use this to optimise - * their own TLB flushing). - * Returns with *start = min(*start, lowest purged address) - * *end = max(*end, highest purged address) */ -static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, - int sync, int force_flush) +static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) { - static DEFINE_SPINLOCK(purge_lock); struct llist_node *valist; struct vmap_area *va; struct vmap_area *n_va; int nr = 0; - /* - * If sync is 0 but force_flush is 1, we'll go sync anyway but callers - * should not expect such behaviour. This just simplifies locking for - * the case that isn't actually used at the moment anyway. - */ - if (!sync && !force_flush) { - if (!spin_trylock(&purge_lock)) - return; - } else - spin_lock(&purge_lock); - - if (sync) - purge_fragmented_blocks_allcpus(); + lockdep_assert_held(&vmap_purge_lock); valist = llist_del_all(&vmap_purge_list); llist_for_each_entry(va, valist, purge_list) { - if (va->va_start < *start) - *start = va->va_start; - if (va->va_end > *end) - *end = va->va_end; + if (va->va_start < start) + start = va->va_start; + if (va->va_end > end) + end = va->va_end; nr += (va->va_end - va->va_start) >> PAGE_SHIFT; } - if (nr) - atomic_sub(nr, &vmap_lazy_nr); + if (!nr) + return false; - if (nr || force_flush) - flush_tlb_kernel_range(*start, *end); + atomic_sub(nr, &vmap_lazy_nr); + flush_tlb_kernel_range(start, end); - if (nr) { - spin_lock(&vmap_area_lock); - llist_for_each_entry_safe(va, n_va, valist, purge_list) - __free_vmap_area(va); - spin_unlock(&vmap_area_lock); - } - spin_unlock(&purge_lock); + spin_lock(&vmap_area_lock); + llist_for_each_entry_safe(va, n_va, valist, purge_list) + __free_vmap_area(va); + spin_unlock(&vmap_area_lock); + return true; } /* @@ -676,9 +660,10 @@ static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, */ static void try_purge_vmap_area_lazy(void) { - unsigned long start = ULONG_MAX, end = 0; - - __purge_vmap_area_lazy(&start, &end, 0, 0); + if (spin_trylock(&vmap_purge_lock)) { + __purge_vmap_area_lazy(ULONG_MAX, 0); + spin_unlock(&vmap_purge_lock); + } } /* @@ -686,9 +671,10 @@ static void try_purge_vmap_area_lazy(void) */ static void purge_vmap_area_lazy(void) { - unsigned long start = ULONG_MAX, end = 0; - - __purge_vmap_area_lazy(&start, &end, 1, 0); + spin_lock(&vmap_purge_lock); + purge_fragmented_blocks_allcpus(); + __purge_vmap_area_lazy(ULONG_MAX, 0); + spin_unlock(&vmap_purge_lock); } /* @@ -1094,7 +1080,11 @@ void vm_unmap_aliases(void) rcu_read_unlock(); } - __purge_vmap_area_lazy(&start, &end, 1, flush); + spin_lock(&vmap_purge_lock); + purge_fragmented_blocks_allcpus(); + if (!__purge_vmap_area_lazy(start, end) && flush) + flush_tlb_kernel_range(start, end); + spin_unlock(&vmap_purge_lock); } EXPORT_SYMBOL_GPL(vm_unmap_aliases); -- 2.1.4
WARNING: multiple messages have this Message-ID (diff)
From: Christoph Hellwig <hch@lst.de> To: akpm@linux-foundation.org Cc: joelaf@google.com, jszhang@marvell.com, chris@chris-wilson.co.uk, joaodias@google.com, linux-mm@kvack.org, linux-rt-users@vger.kernel.org, linux-kernel@vger.kernel.org Subject: [PATCH 1/6] mm: refactor __purge_vmap_area_lazy Date: Tue, 18 Oct 2016 08:56:06 +0200 [thread overview] Message-ID: <1476773771-11470-2-git-send-email-hch@lst.de> (raw) In-Reply-To: <1476773771-11470-1-git-send-email-hch@lst.de> Move the purge_lock synchronization to the callers, move the call to purge_fragmented_blocks_allcpus at the beginning of the function to the callers that need it, move the force_flush behavior to the caller that needs it, and pass start and end by value instead of by reference. No change in behavior. Signed-off-by: Christoph Hellwig <hch@lst.de> --- mm/vmalloc.c | 80 ++++++++++++++++++++++++++---------------------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f2481cb..d045a10 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -601,6 +601,13 @@ static unsigned long lazy_max_pages(void) static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); +/* + * Serialize vmap purging. There is no actual criticial section protected + * by this look, but we want to avoid concurrent calls for performance + * reasons and to make the pcpu_get_vm_areas more deterministic. + */ +static DEFINE_SPINLOCK(vmap_purge_lock); + /* for per-CPU blocks */ static void purge_fragmented_blocks_allcpus(void); @@ -615,59 +622,36 @@ void set_iounmap_nonlazy(void) /* * Purges all lazily-freed vmap areas. - * - * If sync is 0 then don't purge if there is already a purge in progress. - * If force_flush is 1, then flush kernel TLBs between *start and *end even - * if we found no lazy vmap areas to unmap (callers can use this to optimise - * their own TLB flushing). - * Returns with *start = min(*start, lowest purged address) - * *end = max(*end, highest purged address) */ -static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, - int sync, int force_flush) +static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) { - static DEFINE_SPINLOCK(purge_lock); struct llist_node *valist; struct vmap_area *va; struct vmap_area *n_va; int nr = 0; - /* - * If sync is 0 but force_flush is 1, we'll go sync anyway but callers - * should not expect such behaviour. This just simplifies locking for - * the case that isn't actually used at the moment anyway. - */ - if (!sync && !force_flush) { - if (!spin_trylock(&purge_lock)) - return; - } else - spin_lock(&purge_lock); - - if (sync) - purge_fragmented_blocks_allcpus(); + lockdep_assert_held(&vmap_purge_lock); valist = llist_del_all(&vmap_purge_list); llist_for_each_entry(va, valist, purge_list) { - if (va->va_start < *start) - *start = va->va_start; - if (va->va_end > *end) - *end = va->va_end; + if (va->va_start < start) + start = va->va_start; + if (va->va_end > end) + end = va->va_end; nr += (va->va_end - va->va_start) >> PAGE_SHIFT; } - if (nr) - atomic_sub(nr, &vmap_lazy_nr); + if (!nr) + return false; - if (nr || force_flush) - flush_tlb_kernel_range(*start, *end); + atomic_sub(nr, &vmap_lazy_nr); + flush_tlb_kernel_range(start, end); - if (nr) { - spin_lock(&vmap_area_lock); - llist_for_each_entry_safe(va, n_va, valist, purge_list) - __free_vmap_area(va); - spin_unlock(&vmap_area_lock); - } - spin_unlock(&purge_lock); + spin_lock(&vmap_area_lock); + llist_for_each_entry_safe(va, n_va, valist, purge_list) + __free_vmap_area(va); + spin_unlock(&vmap_area_lock); + return true; } /* @@ -676,9 +660,10 @@ static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, */ static void try_purge_vmap_area_lazy(void) { - unsigned long start = ULONG_MAX, end = 0; - - __purge_vmap_area_lazy(&start, &end, 0, 0); + if (spin_trylock(&vmap_purge_lock)) { + __purge_vmap_area_lazy(ULONG_MAX, 0); + spin_unlock(&vmap_purge_lock); + } } /* @@ -686,9 +671,10 @@ static void try_purge_vmap_area_lazy(void) */ static void purge_vmap_area_lazy(void) { - unsigned long start = ULONG_MAX, end = 0; - - __purge_vmap_area_lazy(&start, &end, 1, 0); + spin_lock(&vmap_purge_lock); + purge_fragmented_blocks_allcpus(); + __purge_vmap_area_lazy(ULONG_MAX, 0); + spin_unlock(&vmap_purge_lock); } /* @@ -1094,7 +1080,11 @@ void vm_unmap_aliases(void) rcu_read_unlock(); } - __purge_vmap_area_lazy(&start, &end, 1, flush); + spin_lock(&vmap_purge_lock); + purge_fragmented_blocks_allcpus(); + if (!__purge_vmap_area_lazy(start, end) && flush) + flush_tlb_kernel_range(start, end); + spin_unlock(&vmap_purge_lock); } EXPORT_SYMBOL_GPL(vm_unmap_aliases); -- 2.1.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@kvack.org. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2016-10-18 6:58 UTC|newest] Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top 2016-10-18 6:56 [RFC] reduce latency in __purge_vmap_area_lazy Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig [this message] 2016-10-18 6:56 ` [PATCH 1/6] mm: refactor __purge_vmap_area_lazy Christoph Hellwig 2016-10-18 6:56 ` [PATCH 2/6] mm: mark all calls into the vmalloc subsystem as potentially sleeping Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig 2016-10-18 10:33 ` Chris Wilson 2016-10-18 10:33 ` Chris Wilson 2016-10-18 10:38 ` Christoph Hellwig 2016-10-18 10:38 ` Christoph Hellwig 2016-10-19 11:15 ` Chris Wilson 2016-10-19 11:15 ` Chris Wilson 2016-10-19 13:05 ` Christoph Hellwig 2016-10-19 13:05 ` Christoph Hellwig 2016-10-19 15:34 ` Andy Lutomirski 2016-10-19 15:34 ` Andy Lutomirski 2016-10-19 16:31 ` Christoph Hellwig 2016-10-19 16:31 ` Christoph Hellwig 2016-10-19 19:43 ` Chris Wilson 2016-10-19 19:43 ` Chris Wilson 2016-10-21 0:32 ` Joel Fernandes 2016-10-21 0:32 ` Joel Fernandes 2016-11-08 13:24 ` Joel Fernandes 2016-11-08 13:24 ` Joel Fernandes 2016-11-08 14:32 ` Andrey Ryabinin 2016-11-08 14:32 ` Andrey Ryabinin 2016-10-18 6:56 ` [PATCH 3/6] mm: remove free_unmap_vmap_area_noflush Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig 2016-10-18 6:56 ` [PATCH 4/6] mm: remove free_unmap_vmap_area_addr Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig 2016-10-21 0:46 ` Joel Fernandes 2016-10-21 0:46 ` Joel Fernandes 2016-10-21 1:58 ` Nicholas Piggin 2016-10-21 1:58 ` Nicholas Piggin 2016-10-18 6:56 ` [PATCH 5/6] mm: turn vmap_purge_lock into a mutex Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig 2016-10-18 6:56 ` [PATCH 6/6] mm: add preempt points into __purge_vmap_area_lazy Christoph Hellwig 2016-10-18 6:56 ` Christoph Hellwig 2016-10-18 20:56 ` Steven Rostedt 2016-10-18 20:56 ` Steven Rostedt 2016-10-18 21:01 ` Steven Rostedt 2016-10-18 21:01 ` Steven Rostedt 2016-10-18 10:40 ` [RFC] reduce latency in __purge_vmap_area_lazy Nicholas Piggin 2016-10-18 10:40 ` Nicholas Piggin 2016-10-18 11:21 ` Jisheng Zhang 2016-10-18 11:21 ` Jisheng Zhang 2016-10-18 11:21 ` Jisheng Zhang 2016-10-21 1:08 ` Joel Fernandes 2016-10-21 1:08 ` Joel Fernandes
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=1476773771-11470-2-git-send-email-hch@lst.de \ --to=hch@lst.de \ --cc=akpm@linux-foundation.org \ --cc=chris@chris-wilson.co.uk \ --cc=joaodias@google.com \ --cc=joelaf@google.com \ --cc=jszhang@marvell.com \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=linux-rt-users@vger.kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.