From: "Matthew Wilcox (Oracle)" <willy@infradead.org> To: linux-mm@kvack.org, linux-arch@vger.kernel.org Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>, linux-kernel@vger.kernel.org, Catalin Marinas <catalin.marinas@arm.com>, linux-arm-kernel@lists.infradead.org Subject: [PATCH v3 08/34] arm64: Implement the new page table range API Date: Tue, 28 Feb 2023 21:37:11 +0000 [thread overview] Message-ID: <20230228213738.272178-9-willy@infradead.org> (raw) In-Reply-To: <20230228213738.272178-1-willy@infradead.org> Add set_ptes(), update_mmu_cache_range() and flush_dcache_folio(). Change the PG_dcache_clean flag from being per-page to per-folio. Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> Cc: linux-arm-kernel@lists.infradead.org --- arch/arm64/include/asm/cacheflush.h | 4 +++- arch/arm64/include/asm/pgtable.h | 25 ++++++++++++++------ arch/arm64/mm/flush.c | 36 +++++++++++------------------ 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 37185e978aeb..d115451ed263 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -114,7 +114,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, #define copy_to_user_page copy_to_user_page /* - * flush_dcache_page is used when the kernel has written to the page + * flush_dcache_folio is used when the kernel has written to the page * cache page at virtual address page->virtual. * * If this page isn't mapped (ie, page_mapping == NULL), or it might @@ -127,6 +127,8 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, */ #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 extern void flush_dcache_page(struct page *); +void flush_dcache_folio(struct folio *); +#define flush_dcache_folio flush_dcache_folio static __always_inline void icache_inval_all_pou(void) { diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 69765dc697af..4d1b79dbff16 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -355,12 +355,21 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, set_pte(ptep, pte); } -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte) -{ - page_table_check_ptes_set(mm, addr, ptep, pte, 1); - return __set_pte_at(mm, addr, ptep, pte); +static inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + + for (;;) { + __set_pte_at(mm, addr, ptep, pte); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + pte_val(pte) += PAGE_SIZE; + } } +#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1) /* * Huge pte definitions. @@ -1059,8 +1068,8 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) /* * On AArch64, the cache coherency is handled via the set_pte_at() function. */ -static inline void update_mmu_cache(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static inline void update_mmu_cache_range(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned int nr) { /* * We don't do anything here, so there's a very small chance of @@ -1069,6 +1078,8 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, */ } +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(vma, addr, ptep, 1) #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) #ifdef CONFIG_ARM64_PA_BITS_52 diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 5f9379b3c8c8..deb781af0a3a 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -50,20 +50,13 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, void __sync_icache_dcache(pte_t pte) { - struct page *page = pte_page(pte); + struct folio *folio = page_folio(pte_page(pte)); - /* - * HugeTLB pages are always fully mapped, so only setting head page's - * PG_dcache_clean flag is enough. - */ - if (PageHuge(page)) - page = compound_head(page); - - if (!test_bit(PG_dcache_clean, &page->flags)) { - sync_icache_aliases((unsigned long)page_address(page), - (unsigned long)page_address(page) + - page_size(page)); - set_bit(PG_dcache_clean, &page->flags); + if (!test_bit(PG_dcache_clean, &folio->flags)) { + sync_icache_aliases((unsigned long)folio_address(folio), + (unsigned long)folio_address(folio) + + folio_size(folio)); + set_bit(PG_dcache_clean, &folio->flags); } } EXPORT_SYMBOL_GPL(__sync_icache_dcache); @@ -73,17 +66,16 @@ EXPORT_SYMBOL_GPL(__sync_icache_dcache); * it as dirty for later flushing when mapped in user space (if executable, * see __sync_icache_dcache). */ -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { - /* - * HugeTLB pages are always fully mapped and only head page will be - * set PG_dcache_clean (see comments in __sync_icache_dcache()). - */ - if (PageHuge(page)) - page = compound_head(page); + if (test_bit(PG_dcache_clean, &folio->flags)) + clear_bit(PG_dcache_clean, &folio->flags); +} +EXPORT_SYMBOL(flush_dcache_folio); - if (test_bit(PG_dcache_clean, &page->flags)) - clear_bit(PG_dcache_clean, &page->flags); +void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); } EXPORT_SYMBOL(flush_dcache_page); -- 2.39.1
WARNING: multiple messages have this Message-ID (diff)
From: "Matthew Wilcox (Oracle)" <willy@infradead.org> To: linux-mm@kvack.org, linux-arch@vger.kernel.org Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>, linux-kernel@vger.kernel.org, Catalin Marinas <catalin.marinas@arm.com>, linux-arm-kernel@lists.infradead.org Subject: [PATCH v3 08/34] arm64: Implement the new page table range API Date: Tue, 28 Feb 2023 21:37:11 +0000 [thread overview] Message-ID: <20230228213738.272178-9-willy@infradead.org> (raw) In-Reply-To: <20230228213738.272178-1-willy@infradead.org> Add set_ptes(), update_mmu_cache_range() and flush_dcache_folio(). Change the PG_dcache_clean flag from being per-page to per-folio. Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> Cc: linux-arm-kernel@lists.infradead.org --- arch/arm64/include/asm/cacheflush.h | 4 +++- arch/arm64/include/asm/pgtable.h | 25 ++++++++++++++------ arch/arm64/mm/flush.c | 36 +++++++++++------------------ 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 37185e978aeb..d115451ed263 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -114,7 +114,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, #define copy_to_user_page copy_to_user_page /* - * flush_dcache_page is used when the kernel has written to the page + * flush_dcache_folio is used when the kernel has written to the page * cache page at virtual address page->virtual. * * If this page isn't mapped (ie, page_mapping == NULL), or it might @@ -127,6 +127,8 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, */ #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 extern void flush_dcache_page(struct page *); +void flush_dcache_folio(struct folio *); +#define flush_dcache_folio flush_dcache_folio static __always_inline void icache_inval_all_pou(void) { diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 69765dc697af..4d1b79dbff16 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -355,12 +355,21 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, set_pte(ptep, pte); } -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte) -{ - page_table_check_ptes_set(mm, addr, ptep, pte, 1); - return __set_pte_at(mm, addr, ptep, pte); +static inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + + for (;;) { + __set_pte_at(mm, addr, ptep, pte); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + pte_val(pte) += PAGE_SIZE; + } } +#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1) /* * Huge pte definitions. @@ -1059,8 +1068,8 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) /* * On AArch64, the cache coherency is handled via the set_pte_at() function. */ -static inline void update_mmu_cache(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static inline void update_mmu_cache_range(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned int nr) { /* * We don't do anything here, so there's a very small chance of @@ -1069,6 +1078,8 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, */ } +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(vma, addr, ptep, 1) #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) #ifdef CONFIG_ARM64_PA_BITS_52 diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 5f9379b3c8c8..deb781af0a3a 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -50,20 +50,13 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, void __sync_icache_dcache(pte_t pte) { - struct page *page = pte_page(pte); + struct folio *folio = page_folio(pte_page(pte)); - /* - * HugeTLB pages are always fully mapped, so only setting head page's - * PG_dcache_clean flag is enough. - */ - if (PageHuge(page)) - page = compound_head(page); - - if (!test_bit(PG_dcache_clean, &page->flags)) { - sync_icache_aliases((unsigned long)page_address(page), - (unsigned long)page_address(page) + - page_size(page)); - set_bit(PG_dcache_clean, &page->flags); + if (!test_bit(PG_dcache_clean, &folio->flags)) { + sync_icache_aliases((unsigned long)folio_address(folio), + (unsigned long)folio_address(folio) + + folio_size(folio)); + set_bit(PG_dcache_clean, &folio->flags); } } EXPORT_SYMBOL_GPL(__sync_icache_dcache); @@ -73,17 +66,16 @@ EXPORT_SYMBOL_GPL(__sync_icache_dcache); * it as dirty for later flushing when mapped in user space (if executable, * see __sync_icache_dcache). */ -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { - /* - * HugeTLB pages are always fully mapped and only head page will be - * set PG_dcache_clean (see comments in __sync_icache_dcache()). - */ - if (PageHuge(page)) - page = compound_head(page); + if (test_bit(PG_dcache_clean, &folio->flags)) + clear_bit(PG_dcache_clean, &folio->flags); +} +EXPORT_SYMBOL(flush_dcache_folio); - if (test_bit(PG_dcache_clean, &page->flags)) - clear_bit(PG_dcache_clean, &page->flags); +void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); } EXPORT_SYMBOL(flush_dcache_page); -- 2.39.1 _______________________________________________ linux-arm-kernel mailing list linux-arm-kernel@lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2023-02-28 21:38 UTC|newest] Thread overview: 77+ messages / expand[flat|nested] mbox.gz Atom feed top 2023-02-28 21:37 [PATCH v3 00/34] New page table range API Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 01/34] mm: Convert page_table_check_pte_set() to page_table_check_ptes_set() Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 02/34] mm: Add generic flush_icache_pages() and documentation Matthew Wilcox (Oracle) 2023-03-15 9:27 ` Mike Rapoport 2023-02-28 21:37 ` [PATCH v3 03/34] mm: Add folio_flush_mapping() Matthew Wilcox (Oracle) 2023-03-03 10:33 ` Mike Rapoport 2023-02-28 21:37 ` [PATCH v3 04/34] mm: Remove ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 05/34] alpha: Implement the new page table range API Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 06/34] arc: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 07/34] arm: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) [this message] 2023-02-28 21:37 ` [PATCH v3 08/34] arm64: " Matthew Wilcox (Oracle) 2023-03-09 11:03 ` Ryan Roberts 2023-03-09 11:03 ` Ryan Roberts 2023-02-28 21:37 ` [PATCH v3 09/34] csky: " Matthew Wilcox (Oracle) 2023-03-03 11:40 ` Mike Rapoport 2023-02-28 21:37 ` [PATCH v3 10/34] hexagon: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 11/34] ia64: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) 2023-03-03 11:56 ` Mike Rapoport 2023-03-03 11:56 ` Mike Rapoport 2023-03-03 14:36 ` Matthew Wilcox 2023-03-03 14:36 ` Matthew Wilcox 2023-02-28 21:37 ` [PATCH v3 12/34] loongarch: " Matthew Wilcox (Oracle) 2023-03-01 2:04 ` WANG Xuerui 2023-02-28 21:37 ` [PATCH v3 13/34] m68k: " Matthew Wilcox (Oracle) 2023-03-05 10:16 ` Geert Uytterhoeven 2023-03-05 15:28 ` Matthew Wilcox 2023-03-05 16:48 ` Geert Uytterhoeven 2023-03-05 20:44 ` Michael Schmitz 2023-03-06 7:21 ` Geert Uytterhoeven 2023-03-06 23:01 ` Michael Schmitz 2023-02-28 21:37 ` [PATCH v3 14/34] microblaze: " Matthew Wilcox (Oracle) 2023-03-03 10:53 ` Mike Rapoport 2023-03-03 14:38 ` Matthew Wilcox 2023-02-28 21:37 ` [PATCH v3 15/34] mips: " Matthew Wilcox (Oracle) 2023-03-03 12:24 ` Mike Rapoport 2023-02-28 21:37 ` [PATCH v3 16/34] nios2: " Matthew Wilcox (Oracle) 2023-03-03 12:49 ` Mike Rapoport 2023-02-28 21:37 ` [PATCH v3 17/34] openrisc: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 18/34] parisc: " Matthew Wilcox (Oracle) 2023-03-02 16:43 ` John David Anglin 2023-03-02 20:40 ` John David Anglin 2023-03-04 16:27 ` John David Anglin 2023-02-28 21:37 ` [PATCH v3 19/34] powerpc: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 20/34] riscv: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) 2023-03-15 5:23 ` Palmer Dabbelt 2023-03-15 5:23 ` Palmer Dabbelt 2023-02-28 21:37 ` [PATCH v3 21/34] s390: " Matthew Wilcox (Oracle) 2023-03-02 13:31 ` Gerald Schaefer 2023-02-28 21:37 ` [PATCH v3 22/34] superh: " Matthew Wilcox (Oracle) 2023-03-01 8:06 ` Geert Uytterhoeven 2023-03-01 16:17 ` Matthew Wilcox 2023-02-28 21:37 ` [PATCH v3 23/34] sparc32: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 24/34] sparc64: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 25/34] um: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 26/34] x86: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 27/34] xtensa: " Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 28/34] mm: Remove page_mapping_file() Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 29/34] mm: Rationalise flush_icache_pages() and flush_icache_page() Matthew Wilcox (Oracle) 2023-03-05 9:53 ` Geert Uytterhoeven 2023-02-28 21:37 ` [PATCH v3 30/34] mm: Use flush_icache_pages() in do_set_pmd() Matthew Wilcox (Oracle) 2023-03-03 14:02 ` Mike Rapoport 2023-03-03 16:02 ` Matthew Wilcox 2023-02-28 21:37 ` [PATCH v3 31/34] filemap: Add filemap_map_folio_range() Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 32/34] rmap: add folio_add_file_rmap_range() Matthew Wilcox (Oracle) 2023-03-01 3:04 ` Yin, Fengwei 2023-02-28 21:37 ` [PATCH v3 33/34] mm: Convert do_set_pte() to set_pte_range() Matthew Wilcox (Oracle) 2023-02-28 21:37 ` [PATCH v3 34/34] filemap: Batch PTE mappings Matthew Wilcox (Oracle) 2023-03-03 14:19 ` [PATCH v3 00/34] New page table range API Mike Rapoport 2023-03-05 10:15 ` Geert Uytterhoeven 2023-03-09 11:09 ` Ryan Roberts
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20230228213738.272178-9-willy@infradead.org \ --to=willy@infradead.org \ --cc=catalin.marinas@arm.com \ --cc=linux-arch@vger.kernel.org \ --cc=linux-arm-kernel@lists.infradead.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.