From: Will Deacon <will.deacon@arm.com> To: linux-kernel@vger.kernel.org Cc: peterz@infradead.org, benh@au1.ibm.com, torvalds@linux-foundation.org, npiggin@gmail.com, catalin.marinas@arm.com, linux-arm-kernel@lists.infradead.org, Will Deacon <will.deacon@arm.com> Subject: [RFC PATCH 09/11] asm-generic/tlb: Track which levels of the page tables have been cleared Date: Fri, 24 Aug 2018 16:52:44 +0100 [thread overview] Message-ID: <1535125966-7666-10-git-send-email-will.deacon@arm.com> (raw) In-Reply-To: <1535125966-7666-1-git-send-email-will.deacon@arm.com> It is common for architectures with hugepage support to require only a single TLB invalidation operation per hugepage during unmap(), rather than iterating through the mapping at a PAGE_SIZE increment. Currently, however, the level in the page table where the unmap() operation occurs is not stored in the mmu_gather structure, therefore forcing architectures to issue additional TLB invalidation operations or to give up and over-invalidate by e.g. invalidating the entire TLB. Ideally, we could add an interval rbtree to the mmu_gather structure, which would allow us to associate the correct mapping granule with the various sub-mappings within the range being invalidated. However, this is costly in terms of book-keeping and memory management, so instead we approximate by keeping track of the page table levels that are cleared and provide a means to query the smallest granule required for invalidation. Signed-off-by: Will Deacon <will.deacon@arm.com> --- include/asm-generic/tlb.h | 53 ++++++++++++++++++++++++++++++++++++++++------- mm/memory.c | 4 +++- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index a5caf90264e6..081b105d7215 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -116,6 +116,14 @@ struct mmu_gather { */ unsigned int freed_tables : 1; + /* + * at which levels have we cleared entries? + */ + unsigned int cleared_ptes : 1; + unsigned int cleared_pmds : 1; + unsigned int cleared_puds : 1; + unsigned int cleared_p4ds : 1; + struct mmu_gather_batch *active; struct mmu_gather_batch local; struct page *__pages[MMU_GATHER_BUNDLE]; @@ -150,6 +158,10 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) tlb->end = 0; } tlb->freed_tables = 0; + tlb->cleared_ptes = 0; + tlb->cleared_pmds = 0; + tlb->cleared_puds = 0; + tlb->cleared_p4ds = 0; } static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) @@ -199,6 +211,20 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, } #endif +static inline unsigned long tlb_get_unmap_granule(struct mmu_gather *tlb) +{ + if (tlb->cleared_ptes) + return PAGE_SIZE; + if (tlb->cleared_pmds) + return PMD_SIZE; + if (tlb->cleared_puds) + return PUD_SIZE; + if (tlb->cleared_p4ds) + return P4D_SIZE; + + return PAGE_SIZE; +} + /* * In the case of tlb vma handling, we can optimise these away in the * case where we're doing a full MM flush. When we're doing a munmap, @@ -232,13 +258,19 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->cleared_ptes = 1; \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) -#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ - do { \ - __tlb_adjust_range(tlb, address, huge_page_size(h)); \ - __tlb_remove_tlb_entry(tlb, ptep, address); \ +#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ + do { \ + unsigned long _sz = huge_page_size(h); \ + __tlb_adjust_range(tlb, address, _sz); \ + if (_sz == PMD_SIZE) \ + tlb->cleared_pmds = 1; \ + else if (_sz == PUD_SIZE) \ + tlb->cleared_puds = 1; \ + __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) /** @@ -252,6 +284,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ do { \ __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \ + tlb->cleared_pmds = 1; \ __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \ } while (0) @@ -266,6 +299,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \ + tlb->cleared_puds = 1; \ __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ } while (0) @@ -291,7 +325,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pte_free_tlb(tlb, ptep, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ + tlb->cleared_pmds = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) #endif @@ -300,7 +335,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pmd_free_tlb(tlb, pmdp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ + tlb->cleared_puds = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) #endif @@ -310,7 +346,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pud_free_tlb(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ + tlb->cleared_p4ds = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif @@ -321,7 +358,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define p4d_free_tlb(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) #endif diff --git a/mm/memory.c b/mm/memory.c index 83aef222f11b..69982d1d9b7a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -267,8 +267,10 @@ void arch_tlb_finish_mmu(struct mmu_gather *tlb, { struct mmu_gather_batch *batch, *next; - if (force) + if (force) { + __tlb_reset_range(tlb); __tlb_adjust_range(tlb, start, end - start); + } tlb_flush_mmu(tlb); -- 2.1.4
WARNING: multiple messages have this Message-ID (diff)
From: will.deacon@arm.com (Will Deacon) To: linux-arm-kernel@lists.infradead.org Subject: [RFC PATCH 09/11] asm-generic/tlb: Track which levels of the page tables have been cleared Date: Fri, 24 Aug 2018 16:52:44 +0100 [thread overview] Message-ID: <1535125966-7666-10-git-send-email-will.deacon@arm.com> (raw) In-Reply-To: <1535125966-7666-1-git-send-email-will.deacon@arm.com> It is common for architectures with hugepage support to require only a single TLB invalidation operation per hugepage during unmap(), rather than iterating through the mapping at a PAGE_SIZE increment. Currently, however, the level in the page table where the unmap() operation occurs is not stored in the mmu_gather structure, therefore forcing architectures to issue additional TLB invalidation operations or to give up and over-invalidate by e.g. invalidating the entire TLB. Ideally, we could add an interval rbtree to the mmu_gather structure, which would allow us to associate the correct mapping granule with the various sub-mappings within the range being invalidated. However, this is costly in terms of book-keeping and memory management, so instead we approximate by keeping track of the page table levels that are cleared and provide a means to query the smallest granule required for invalidation. Signed-off-by: Will Deacon <will.deacon@arm.com> --- include/asm-generic/tlb.h | 53 ++++++++++++++++++++++++++++++++++++++++------- mm/memory.c | 4 +++- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index a5caf90264e6..081b105d7215 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -116,6 +116,14 @@ struct mmu_gather { */ unsigned int freed_tables : 1; + /* + * at which levels have we cleared entries? + */ + unsigned int cleared_ptes : 1; + unsigned int cleared_pmds : 1; + unsigned int cleared_puds : 1; + unsigned int cleared_p4ds : 1; + struct mmu_gather_batch *active; struct mmu_gather_batch local; struct page *__pages[MMU_GATHER_BUNDLE]; @@ -150,6 +158,10 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) tlb->end = 0; } tlb->freed_tables = 0; + tlb->cleared_ptes = 0; + tlb->cleared_pmds = 0; + tlb->cleared_puds = 0; + tlb->cleared_p4ds = 0; } static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) @@ -199,6 +211,20 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, } #endif +static inline unsigned long tlb_get_unmap_granule(struct mmu_gather *tlb) +{ + if (tlb->cleared_ptes) + return PAGE_SIZE; + if (tlb->cleared_pmds) + return PMD_SIZE; + if (tlb->cleared_puds) + return PUD_SIZE; + if (tlb->cleared_p4ds) + return P4D_SIZE; + + return PAGE_SIZE; +} + /* * In the case of tlb vma handling, we can optimise these away in the * case where we're doing a full MM flush. When we're doing a munmap, @@ -232,13 +258,19 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb->cleared_ptes = 1; \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) -#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ - do { \ - __tlb_adjust_range(tlb, address, huge_page_size(h)); \ - __tlb_remove_tlb_entry(tlb, ptep, address); \ +#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ + do { \ + unsigned long _sz = huge_page_size(h); \ + __tlb_adjust_range(tlb, address, _sz); \ + if (_sz == PMD_SIZE) \ + tlb->cleared_pmds = 1; \ + else if (_sz == PUD_SIZE) \ + tlb->cleared_puds = 1; \ + __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) /** @@ -252,6 +284,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ do { \ __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \ + tlb->cleared_pmds = 1; \ __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \ } while (0) @@ -266,6 +299,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \ + tlb->cleared_puds = 1; \ __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ } while (0) @@ -291,7 +325,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pte_free_tlb(tlb, ptep, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ + tlb->cleared_pmds = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) #endif @@ -300,7 +335,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pmd_free_tlb(tlb, pmdp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ + tlb->cleared_puds = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) #endif @@ -310,7 +346,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define pud_free_tlb(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ + tlb->cleared_p4ds = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif @@ -321,7 +358,7 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, #define p4d_free_tlb(tlb, pudp, address) \ do { \ __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->freed_tables = 1; \ + tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) #endif diff --git a/mm/memory.c b/mm/memory.c index 83aef222f11b..69982d1d9b7a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -267,8 +267,10 @@ void arch_tlb_finish_mmu(struct mmu_gather *tlb, { struct mmu_gather_batch *batch, *next; - if (force) + if (force) { + __tlb_reset_range(tlb); __tlb_adjust_range(tlb, start, end - start); + } tlb_flush_mmu(tlb); -- 2.1.4
next prev parent reply other threads:[~2018-08-24 15:53 UTC|newest] Thread overview: 58+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-08-24 15:52 [RFC PATCH 00/11] Avoid synchronous TLB invalidation for intermediate page-table entries on arm64 Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 01/11] arm64: tlb: Use last-level invalidation in flush_tlb_kernel_range() Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 02/11] arm64: tlb: Add DSB ISHST prior to TLBI in __flush_tlb_[kernel_]pgtable() Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 17:56 ` Peter Zijlstra 2018-08-24 17:56 ` Peter Zijlstra 2018-08-28 13:03 ` Will Deacon 2018-08-28 13:03 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 03/11] arm64: pgtable: Implement p[mu]d_valid() and check in set_p[mu]d() Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 16:15 ` Linus Torvalds 2018-08-24 16:15 ` Linus Torvalds 2018-08-28 12:49 ` Will Deacon 2018-08-28 12:49 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 04/11] arm64: tlb: Justify non-leaf invalidation in flush_tlb_range() Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 05/11] arm64: tlbflush: Allow stride to be specified for __flush_tlb_range() Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 06/11] arm64: tlb: Remove redundant !CONFIG_HAVE_RCU_TABLE_FREE code Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 07/11] asm-generic/tlb: Guard with #ifdef CONFIG_MMU Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 08/11] asm-generic/tlb: Track freeing of page-table directories in struct mmu_gather Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-27 4:44 ` Nicholas Piggin 2018-08-27 4:44 ` Nicholas Piggin 2018-08-28 13:46 ` Peter Zijlstra 2018-08-28 13:46 ` Peter Zijlstra 2018-08-28 13:48 ` Peter Zijlstra 2018-08-28 13:48 ` Peter Zijlstra 2018-08-28 14:12 ` Nicholas Piggin 2018-08-28 14:12 ` Nicholas Piggin 2018-08-24 15:52 ` Will Deacon [this message] 2018-08-24 15:52 ` [RFC PATCH 09/11] asm-generic/tlb: Track which levels of the page tables have been cleared Will Deacon 2018-08-27 7:53 ` Peter Zijlstra 2018-08-27 7:53 ` Peter Zijlstra 2018-08-28 13:12 ` Will Deacon 2018-08-28 13:12 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 10/11] arm64: tlb: Adjust stride and type of TLBI according to mmu_gather Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 15:52 ` [RFC PATCH 11/11] arm64: tlb: Avoid synchronous TLBIs when freeing page tables Will Deacon 2018-08-24 15:52 ` Will Deacon 2018-08-24 16:20 ` [RFC PATCH 00/11] Avoid synchronous TLB invalidation for intermediate page-table entries on arm64 Linus Torvalds 2018-08-24 16:20 ` Linus Torvalds 2018-08-26 10:56 ` Peter Zijlstra 2018-08-26 10:56 ` Peter Zijlstra 2018-09-04 18:38 ` Jon Masters 2018-09-04 18:38 ` Jon Masters 2018-09-05 12:28 ` Will Deacon 2018-09-05 12:28 ` Will Deacon 2018-09-07 6:36 ` Jon Masters 2018-09-07 6:36 ` Jon Masters 2018-09-13 15:53 ` Will Deacon 2018-09-13 15:53 ` Will Deacon 2018-09-13 16:53 ` Jon Masters 2018-09-13 16:53 ` Jon Masters
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=1535125966-7666-10-git-send-email-will.deacon@arm.com \ --to=will.deacon@arm.com \ --cc=benh@au1.ibm.com \ --cc=catalin.marinas@arm.com \ --cc=linux-arm-kernel@lists.infradead.org \ --cc=linux-kernel@vger.kernel.org \ --cc=npiggin@gmail.com \ --cc=peterz@infradead.org \ --cc=torvalds@linux-foundation.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.