* [PATCH 1/3] MIPS: Fix cache flushing for swap pages with non-DMA I/O.
2015-01-29 16:46 [PATCH 0/3] HIGHMEM and cache flush fixes Steven J. Hill
@ 2015-01-29 16:46 ` Steven J. Hill
2015-02-04 14:54 ` Lars Persson
2015-01-29 16:46 ` [PATCH 2/3] MIPS: Highmem: Fixes for cache aliasing and color Steven J. Hill
2015-01-29 16:46 ` [PATCH 3/3] MIPS: Fix I-cache flushing for kmap'd pages Steven J. Hill
2 siblings, 1 reply; 6+ messages in thread
From: Steven J. Hill @ 2015-01-29 16:46 UTC (permalink / raw)
To: linux-mips; +Cc: ralf
From: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
Flush the D-cache before the page is given to a process
as an executable (I-cache) page when the backing store
is non-DMA I/O.
Signed-off-by: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
Signed-off-by: Steven J. Hill <Steven.Hill@imgtec.com>
---
arch/mips/include/asm/cacheflush.h | 3 ++-
arch/mips/include/asm/cpu-features.h | 3 +++
arch/mips/include/asm/page.h | 5 +++-
arch/mips/include/asm/pgtable.h | 5 ++++
arch/mips/mm/c-r4k.c | 19 +++++++++++++-
arch/mips/mm/cache.c | 46 +++++++++++++++++-----------------
arch/mips/mm/init.c | 23 +++++++++++------
arch/mips/mm/sc-mips.c | 1 +
8 files changed, 72 insertions(+), 33 deletions(-)
diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h
index e08381a..3a4582a 100644
--- a/arch/mips/include/asm/cacheflush.h
+++ b/arch/mips/include/asm/cacheflush.h
@@ -123,7 +123,8 @@ static inline void kunmap_noncoherent(void)
#define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
static inline void flush_kernel_dcache_page(struct page *page)
{
- BUG_ON(cpu_has_dc_aliases && PageHighMem(page));
+ if (cpu_has_dc_aliases || !cpu_has_ic_fills_f_dc)
+ __flush_dcache_page(page);
}
/*
diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
index 2897cfa..23db770 100644
--- a/arch/mips/include/asm/cpu-features.h
+++ b/arch/mips/include/asm/cpu-features.h
@@ -139,6 +139,9 @@
#ifndef cpu_has_vtag_icache
#define cpu_has_vtag_icache (cpu_data[0].icache.flags & MIPS_CACHE_VTAG)
#endif
+#ifndef cpu_has_vtag_dcache
+#define cpu_has_vtag_dcache (cpu_data[0].dcache.flags & MIPS_CACHE_VTAG)
+#endif
#ifndef cpu_has_dc_aliases
#define cpu_has_dc_aliases (cpu_data[0].dcache.flags & MIPS_CACHE_ALIASES)
#endif
diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h
index 154b70a..fc7ab98 100644
--- a/arch/mips/include/asm/page.h
+++ b/arch/mips/include/asm/page.h
@@ -95,13 +95,16 @@ static inline unsigned long pages_do_alias(unsigned long addr1,
struct page;
+#include <asm/cpu-features.h>
+
static inline void clear_user_page(void *addr, unsigned long vaddr,
struct page *page)
{
extern void (*flush_data_cache_page)(unsigned long addr);
clear_page(addr);
- if (pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK))
+ if (cpu_has_vtag_dcache || (cpu_has_dc_aliases &&
+ pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK)))
flush_data_cache_page((unsigned long)addr);
}
diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
index 62a6ba3..0a6a944 100644
--- a/arch/mips/include/asm/pgtable.h
+++ b/arch/mips/include/asm/pgtable.h
@@ -148,6 +148,7 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
}
}
}
+#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
@@ -185,6 +186,7 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
}
#endif
}
+#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
@@ -401,12 +403,15 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
extern void __update_tlb(struct vm_area_struct *vma, unsigned long address,
pte_t pte);
+extern void __update_cache(struct vm_area_struct *vma, unsigned long address,
+ pte_t pte);
static inline void update_mmu_cache(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
pte_t pte = *ptep;
__update_tlb(vma, address, pte);
+ __update_cache(vma, address, pte);
}
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index dd261df..e045116 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -484,8 +484,11 @@ static inline void local_r4k_flush_cache_range(void * args)
return;
r4k_blast_dcache();
- if (exec)
+ if (exec) {
+ if (!cpu_has_ic_fills_f_dc)
+ wmb();
r4k_blast_icache();
+ }
}
static void r4k_flush_cache_range(struct vm_area_struct *vma,
@@ -570,6 +573,14 @@ static inline void local_r4k_flush_cache_page(void *args)
if (!(pte_present(*ptep)))
return;
+ /*
+ * If this page is not destined to be executable and the
+ * data cache does not have aliases, all of the mapping
+ * below can be skipped.
+ */
+ if (!exec && !cpu_has_dc_aliases)
+ return;
+
if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
vaddr = NULL;
else {
@@ -589,6 +600,8 @@ static inline void local_r4k_flush_cache_page(void *args)
if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
vaddr ? r4k_blast_dcache_page(addr) :
r4k_blast_dcache_user_page(addr);
+ if (exec && !cpu_has_ic_fills_f_dc)
+ wmb();
if (exec && !cpu_icache_snoops_remote_store)
r4k_blast_scache_page(addr);
}
@@ -621,6 +634,8 @@ static void r4k_flush_cache_page(struct vm_area_struct *vma,
args.pfn = pfn;
r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
+ if (cpu_has_dc_aliases)
+ ClearPageDcacheDirty(pfn_to_page(pfn));
}
static inline void local_r4k_flush_data_cache_page(void * addr)
@@ -652,6 +667,8 @@ static inline void local_r4k_flush_icache_range(unsigned long start, unsigned lo
}
}
+ wmb();
+
if (end - start > icache_size)
r4k_blast_icache();
else {
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
index 7e3ea77..99db9e8 100644
--- a/arch/mips/mm/cache.c
+++ b/arch/mips/mm/cache.c
@@ -19,6 +19,7 @@
#include <asm/processor.h>
#include <asm/cpu.h>
#include <asm/cpu-features.h>
+#include <linux/highmem.h>
/* Cache operations. */
void (*flush_cache_all)(void);
@@ -105,48 +106,47 @@ void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
unsigned long addr = (unsigned long) page_address(page);
- if (pages_do_alias(addr, vmaddr)) {
+ if (pages_do_alias(addr, vmaddr & PAGE_MASK)) {
if (page_mapped(page) && !Page_dcache_dirty(page)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
flush_data_cache_page((unsigned long)kaddr);
kunmap_coherent();
- } else
- flush_data_cache_page(addr);
+ } else {
+ void *kaddr;
+
+ kaddr = kmap_atomic(page);
+ flush_data_cache_page((unsigned long)kaddr);
+ kunmap_atomic(kaddr);
+ ClearPageDcacheDirty(page);
+ }
}
}
EXPORT_SYMBOL(__flush_anon_page);
-static void mips_flush_dcache_from_pte(pte_t pteval, unsigned long address)
+void __update_cache(struct vm_area_struct *vma, unsigned long address,
+ pte_t pte)
{
struct page *page;
- unsigned long pfn = pte_pfn(pteval);
+ unsigned long pfn = pte_pfn(pte);
+ int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc;
- if (unlikely(!pfn_valid(pfn)))
+ if (unlikely(!pfn_valid(pfn))) {
+ wmb();
return;
-
+ }
page = pfn_to_page(pfn);
- if (page_mapping(page) && Page_dcache_dirty(page)) {
+ if (page_mapped(page) && Page_dcache_dirty(page)) {
unsigned long page_addr = (unsigned long) page_address(page);
-
- if (!cpu_has_ic_fills_f_dc ||
- pages_do_alias(page_addr, address & PAGE_MASK))
+ if (exec || (cpu_has_dc_aliases &&
+ pages_do_alias(page_addr, address & PAGE_MASK))) {
flush_data_cache_page(page_addr);
- ClearPageDcacheDirty(page);
+ ClearPageDcacheDirty(page);
+ }
}
-}
-
-void set_pte_at(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pteval)
-{
- if (cpu_has_dc_aliases || !cpu_has_ic_fills_f_dc) {
- if (pte_present(pteval))
- mips_flush_dcache_from_pte(pteval, addr);
- }
-
- set_pte(ptep, pteval);
+ wmb(); /* finish any outstanding arch cache flushes before ret to user */
}
unsigned long _page_cachable_default;
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 448cde3..597bf7f 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -165,9 +165,15 @@ void copy_user_highpage(struct page *to, struct page *from,
copy_page(vto, vfrom);
kunmap_atomic(vfrom);
}
- if ((!cpu_has_ic_fills_f_dc) ||
- pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
+ if (cpu_has_dc_aliases)
+ SetPageDcacheDirty(to);
+ if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
+ cpu_has_vtag_dcache || (cpu_has_dc_aliases &&
+ pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))) {
flush_data_cache_page((unsigned long)vto);
+ if (cpu_has_dc_aliases)
+ ClearPageDcacheDirty(to);
+ }
kunmap_atomic(vto);
/* Make sure this page is cleared on other CPU's too before using it */
smp_wmb();
@@ -187,8 +193,14 @@ void copy_to_user_page(struct vm_area_struct *vma,
if (cpu_has_dc_aliases)
SetPageDcacheDirty(page);
}
- if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
+ if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
+ (Page_dcache_dirty(page) &&
+ pages_do_alias((unsigned long)dst & PAGE_MASK,
+ vaddr & PAGE_MASK))) {
flush_cache_page(vma, vaddr, page_to_pfn(page));
+ if (cpu_has_dc_aliases)
+ ClearPageDcacheDirty(page);
+ }
}
void copy_from_user_page(struct vm_area_struct *vma,
@@ -200,11 +212,8 @@ void copy_from_user_page(struct vm_area_struct *vma,
void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
memcpy(dst, vfrom, len);
kunmap_coherent();
- } else {
+ } else
memcpy(dst, src, len);
- if (cpu_has_dc_aliases)
- SetPageDcacheDirty(page);
- }
}
EXPORT_SYMBOL_GPL(copy_from_user_page);
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
index 99eb8fa..e1db410 100644
--- a/arch/mips/mm/sc-mips.c
+++ b/arch/mips/mm/sc-mips.c
@@ -24,6 +24,7 @@
*/
static void mips_sc_wback_inv(unsigned long addr, unsigned long size)
{
+ __sync();
blast_scache_range(addr, addr + size);
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [1/3] MIPS: Fix cache flushing for swap pages with non-DMA I/O.
@ 2015-02-04 14:54 ` Lars Persson
0 siblings, 0 replies; 6+ messages in thread
From: Lars Persson @ 2015-02-04 14:54 UTC (permalink / raw)
To: Steven J. Hill; +Cc: linux-mips, ralf
Hi
I NACK this patch series because AFAICT it re-introduces the race
condition that I fixed with 2a4a8b1e5d9d343e13ff22e19af7b353f7b52d6f
MIPS: Remove race window in page fault handling.
You need to address also the case of the file-system layer writing to an
executable page and calling flush_dcache_page().
With this patch series, the flush will be postponed by this check:
if (page_mapping(page) && !page_mapped(page)) {
SetPageDcacheDirty(page);
return;
}
Another thread can race with __update_cache() and execute garbage code.
Please consider adding a commit of the postponed cache flush in
flush_icache_page().
BR,
Lars
On tor, 2015-01-29 at 10:46 -0600, Steven J. Hill wrote:
> From: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
>
> Flush the D-cache before the page is given to a process
> as an executable (I-cache) page when the backing store
> is non-DMA I/O.
>
> Signed-off-by: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
> Signed-off-by: Steven J. Hill <Steven.Hill@imgtec.com>
> ---
> arch/mips/include/asm/cacheflush.h | 3 ++-
> arch/mips/include/asm/cpu-features.h | 3 +++
> arch/mips/include/asm/page.h | 5 +++-
> arch/mips/include/asm/pgtable.h | 5 ++++
> arch/mips/mm/c-r4k.c | 19 +++++++++++++-
> arch/mips/mm/cache.c | 46 +++++++++++++++++-----------------
> arch/mips/mm/init.c | 23 +++++++++++------
> arch/mips/mm/sc-mips.c | 1 +
> 8 files changed, 72 insertions(+), 33 deletions(-)
>
> diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h
> index e08381a..3a4582a 100644
> --- a/arch/mips/include/asm/cacheflush.h
> +++ b/arch/mips/include/asm/cacheflush.h
> @@ -123,7 +123,8 @@ static inline void kunmap_noncoherent(void)
> #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
> static inline void flush_kernel_dcache_page(struct page *page)
> {
> - BUG_ON(cpu_has_dc_aliases && PageHighMem(page));
> + if (cpu_has_dc_aliases || !cpu_has_ic_fills_f_dc)
> + __flush_dcache_page(page);
> }
>
> /*
> diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
> index 2897cfa..23db770 100644
> --- a/arch/mips/include/asm/cpu-features.h
> +++ b/arch/mips/include/asm/cpu-features.h
> @@ -139,6 +139,9 @@
> #ifndef cpu_has_vtag_icache
> #define cpu_has_vtag_icache (cpu_data[0].icache.flags & MIPS_CACHE_VTAG)
> #endif
> +#ifndef cpu_has_vtag_dcache
> +#define cpu_has_vtag_dcache (cpu_data[0].dcache.flags & MIPS_CACHE_VTAG)
> +#endif
> #ifndef cpu_has_dc_aliases
> #define cpu_has_dc_aliases (cpu_data[0].dcache.flags & MIPS_CACHE_ALIASES)
> #endif
> diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h
> index 154b70a..fc7ab98 100644
> --- a/arch/mips/include/asm/page.h
> +++ b/arch/mips/include/asm/page.h
> @@ -95,13 +95,16 @@ static inline unsigned long pages_do_alias(unsigned long addr1,
>
> struct page;
>
> +#include <asm/cpu-features.h>
> +
> static inline void clear_user_page(void *addr, unsigned long vaddr,
> struct page *page)
> {
> extern void (*flush_data_cache_page)(unsigned long addr);
>
> clear_page(addr);
> - if (pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK))
> + if (cpu_has_vtag_dcache || (cpu_has_dc_aliases &&
> + pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK)))
> flush_data_cache_page((unsigned long)addr);
> }
>
> diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
> index 62a6ba3..0a6a944 100644
> --- a/arch/mips/include/asm/pgtable.h
> +++ b/arch/mips/include/asm/pgtable.h
> @@ -148,6 +148,7 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
> }
> }
> }
> +#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
>
> static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
> {
> @@ -185,6 +186,7 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
> }
> #endif
> }
> +#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
>
> static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
> {
> @@ -401,12 +403,15 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
>
> extern void __update_tlb(struct vm_area_struct *vma, unsigned long address,
> pte_t pte);
> +extern void __update_cache(struct vm_area_struct *vma, unsigned long address,
> + pte_t pte);
>
> static inline void update_mmu_cache(struct vm_area_struct *vma,
> unsigned long address, pte_t *ptep)
> {
> pte_t pte = *ptep;
> __update_tlb(vma, address, pte);
> + __update_cache(vma, address, pte);
> }
>
> static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
> diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
> index dd261df..e045116 100644
> --- a/arch/mips/mm/c-r4k.c
> +++ b/arch/mips/mm/c-r4k.c
> @@ -484,8 +484,11 @@ static inline void local_r4k_flush_cache_range(void * args)
> return;
>
> r4k_blast_dcache();
> - if (exec)
> + if (exec) {
> + if (!cpu_has_ic_fills_f_dc)
> + wmb();
> r4k_blast_icache();
> + }
> }
>
> static void r4k_flush_cache_range(struct vm_area_struct *vma,
> @@ -570,6 +573,14 @@ static inline void local_r4k_flush_cache_page(void *args)
> if (!(pte_present(*ptep)))
> return;
>
> + /*
> + * If this page is not destined to be executable and the
> + * data cache does not have aliases, all of the mapping
> + * below can be skipped.
> + */
> + if (!exec && !cpu_has_dc_aliases)
> + return;
> +
> if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
> vaddr = NULL;
> else {
> @@ -589,6 +600,8 @@ static inline void local_r4k_flush_cache_page(void *args)
> if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
> vaddr ? r4k_blast_dcache_page(addr) :
> r4k_blast_dcache_user_page(addr);
> + if (exec && !cpu_has_ic_fills_f_dc)
> + wmb();
> if (exec && !cpu_icache_snoops_remote_store)
> r4k_blast_scache_page(addr);
> }
> @@ -621,6 +634,8 @@ static void r4k_flush_cache_page(struct vm_area_struct *vma,
> args.pfn = pfn;
>
> r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
> + if (cpu_has_dc_aliases)
> + ClearPageDcacheDirty(pfn_to_page(pfn));
> }
>
> static inline void local_r4k_flush_data_cache_page(void * addr)
> @@ -652,6 +667,8 @@ static inline void local_r4k_flush_icache_range(unsigned long start, unsigned lo
> }
> }
>
> + wmb();
> +
> if (end - start > icache_size)
> r4k_blast_icache();
> else {
> diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
> index 7e3ea77..99db9e8 100644
> --- a/arch/mips/mm/cache.c
> +++ b/arch/mips/mm/cache.c
> @@ -19,6 +19,7 @@
> #include <asm/processor.h>
> #include <asm/cpu.h>
> #include <asm/cpu-features.h>
> +#include <linux/highmem.h>
>
> /* Cache operations. */
> void (*flush_cache_all)(void);
> @@ -105,48 +106,47 @@ void __flush_anon_page(struct page *page, unsigned long vmaddr)
> {
> unsigned long addr = (unsigned long) page_address(page);
>
> - if (pages_do_alias(addr, vmaddr)) {
> + if (pages_do_alias(addr, vmaddr & PAGE_MASK)) {
> if (page_mapped(page) && !Page_dcache_dirty(page)) {
> void *kaddr;
>
> kaddr = kmap_coherent(page, vmaddr);
> flush_data_cache_page((unsigned long)kaddr);
> kunmap_coherent();
> - } else
> - flush_data_cache_page(addr);
> + } else {
> + void *kaddr;
> +
> + kaddr = kmap_atomic(page);
> + flush_data_cache_page((unsigned long)kaddr);
> + kunmap_atomic(kaddr);
> + ClearPageDcacheDirty(page);
> + }
> }
> }
>
> EXPORT_SYMBOL(__flush_anon_page);
>
> -static void mips_flush_dcache_from_pte(pte_t pteval, unsigned long address)
> +void __update_cache(struct vm_area_struct *vma, unsigned long address,
> + pte_t pte)
> {
> struct page *page;
> - unsigned long pfn = pte_pfn(pteval);
> + unsigned long pfn = pte_pfn(pte);
> + int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc;
>
> - if (unlikely(!pfn_valid(pfn)))
> + if (unlikely(!pfn_valid(pfn))) {
> + wmb();
> return;
> -
> + }
> page = pfn_to_page(pfn);
> - if (page_mapping(page) && Page_dcache_dirty(page)) {
> + if (page_mapped(page) && Page_dcache_dirty(page)) {
> unsigned long page_addr = (unsigned long) page_address(page);
> -
> - if (!cpu_has_ic_fills_f_dc ||
> - pages_do_alias(page_addr, address & PAGE_MASK))
> + if (exec || (cpu_has_dc_aliases &&
> + pages_do_alias(page_addr, address & PAGE_MASK))) {
> flush_data_cache_page(page_addr);
> - ClearPageDcacheDirty(page);
> + ClearPageDcacheDirty(page);
> + }
> }
> -}
> -
> -void set_pte_at(struct mm_struct *mm, unsigned long addr,
> - pte_t *ptep, pte_t pteval)
> -{
> - if (cpu_has_dc_aliases || !cpu_has_ic_fills_f_dc) {
> - if (pte_present(pteval))
> - mips_flush_dcache_from_pte(pteval, addr);
> - }
> -
> - set_pte(ptep, pteval);
> + wmb(); /* finish any outstanding arch cache flushes before ret to user */
> }
>
> unsigned long _page_cachable_default;
> diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
> index 448cde3..597bf7f 100644
> --- a/arch/mips/mm/init.c
> +++ b/arch/mips/mm/init.c
> @@ -165,9 +165,15 @@ void copy_user_highpage(struct page *to, struct page *from,
> copy_page(vto, vfrom);
> kunmap_atomic(vfrom);
> }
> - if ((!cpu_has_ic_fills_f_dc) ||
> - pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
> + if (cpu_has_dc_aliases)
> + SetPageDcacheDirty(to);
> + if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
> + cpu_has_vtag_dcache || (cpu_has_dc_aliases &&
> + pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))) {
> flush_data_cache_page((unsigned long)vto);
> + if (cpu_has_dc_aliases)
> + ClearPageDcacheDirty(to);
> + }
> kunmap_atomic(vto);
> /* Make sure this page is cleared on other CPU's too before using it */
> smp_wmb();
> @@ -187,8 +193,14 @@ void copy_to_user_page(struct vm_area_struct *vma,
> if (cpu_has_dc_aliases)
> SetPageDcacheDirty(page);
> }
> - if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
> + if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
> + (Page_dcache_dirty(page) &&
> + pages_do_alias((unsigned long)dst & PAGE_MASK,
> + vaddr & PAGE_MASK))) {
> flush_cache_page(vma, vaddr, page_to_pfn(page));
> + if (cpu_has_dc_aliases)
> + ClearPageDcacheDirty(page);
> + }
> }
>
> void copy_from_user_page(struct vm_area_struct *vma,
> @@ -200,11 +212,8 @@ void copy_from_user_page(struct vm_area_struct *vma,
> void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
> memcpy(dst, vfrom, len);
> kunmap_coherent();
> - } else {
> + } else
> memcpy(dst, src, len);
> - if (cpu_has_dc_aliases)
> - SetPageDcacheDirty(page);
> - }
> }
> EXPORT_SYMBOL_GPL(copy_from_user_page);
>
> diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
> index 99eb8fa..e1db410 100644
> --- a/arch/mips/mm/sc-mips.c
> +++ b/arch/mips/mm/sc-mips.c
> @@ -24,6 +24,7 @@
> */
> static void mips_sc_wback_inv(unsigned long addr, unsigned long size)
> {
> + __sync();
> blast_scache_range(addr, addr + size);
> }
>
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [1/3] MIPS: Fix cache flushing for swap pages with non-DMA I/O.
@ 2015-02-04 14:54 ` Lars Persson
0 siblings, 0 replies; 6+ messages in thread
From: Lars Persson @ 2015-02-04 14:54 UTC (permalink / raw)
To: Steven J. Hill; +Cc: linux-mips, ralf
Hi
I NACK this patch series because AFAICT it re-introduces the race
condition that I fixed with 2a4a8b1e5d9d343e13ff22e19af7b353f7b52d6f
MIPS: Remove race window in page fault handling.
You need to address also the case of the file-system layer writing to an
executable page and calling flush_dcache_page().
With this patch series, the flush will be postponed by this check:
if (page_mapping(page) && !page_mapped(page)) {
SetPageDcacheDirty(page);
return;
}
Another thread can race with __update_cache() and execute garbage code.
Please consider adding a commit of the postponed cache flush in
flush_icache_page().
BR,
Lars
On tor, 2015-01-29 at 10:46 -0600, Steven J. Hill wrote:
> From: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
>
> Flush the D-cache before the page is given to a process
> as an executable (I-cache) page when the backing store
> is non-DMA I/O.
>
> Signed-off-by: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
> Signed-off-by: Steven J. Hill <Steven.Hill@imgtec.com>
> ---
> arch/mips/include/asm/cacheflush.h | 3 ++-
> arch/mips/include/asm/cpu-features.h | 3 +++
> arch/mips/include/asm/page.h | 5 +++-
> arch/mips/include/asm/pgtable.h | 5 ++++
> arch/mips/mm/c-r4k.c | 19 +++++++++++++-
> arch/mips/mm/cache.c | 46 +++++++++++++++++-----------------
> arch/mips/mm/init.c | 23 +++++++++++------
> arch/mips/mm/sc-mips.c | 1 +
> 8 files changed, 72 insertions(+), 33 deletions(-)
>
> diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h
> index e08381a..3a4582a 100644
> --- a/arch/mips/include/asm/cacheflush.h
> +++ b/arch/mips/include/asm/cacheflush.h
> @@ -123,7 +123,8 @@ static inline void kunmap_noncoherent(void)
> #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
> static inline void flush_kernel_dcache_page(struct page *page)
> {
> - BUG_ON(cpu_has_dc_aliases && PageHighMem(page));
> + if (cpu_has_dc_aliases || !cpu_has_ic_fills_f_dc)
> + __flush_dcache_page(page);
> }
>
> /*
> diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
> index 2897cfa..23db770 100644
> --- a/arch/mips/include/asm/cpu-features.h
> +++ b/arch/mips/include/asm/cpu-features.h
> @@ -139,6 +139,9 @@
> #ifndef cpu_has_vtag_icache
> #define cpu_has_vtag_icache (cpu_data[0].icache.flags & MIPS_CACHE_VTAG)
> #endif
> +#ifndef cpu_has_vtag_dcache
> +#define cpu_has_vtag_dcache (cpu_data[0].dcache.flags & MIPS_CACHE_VTAG)
> +#endif
> #ifndef cpu_has_dc_aliases
> #define cpu_has_dc_aliases (cpu_data[0].dcache.flags & MIPS_CACHE_ALIASES)
> #endif
> diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h
> index 154b70a..fc7ab98 100644
> --- a/arch/mips/include/asm/page.h
> +++ b/arch/mips/include/asm/page.h
> @@ -95,13 +95,16 @@ static inline unsigned long pages_do_alias(unsigned long addr1,
>
> struct page;
>
> +#include <asm/cpu-features.h>
> +
> static inline void clear_user_page(void *addr, unsigned long vaddr,
> struct page *page)
> {
> extern void (*flush_data_cache_page)(unsigned long addr);
>
> clear_page(addr);
> - if (pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK))
> + if (cpu_has_vtag_dcache || (cpu_has_dc_aliases &&
> + pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK)))
> flush_data_cache_page((unsigned long)addr);
> }
>
> diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
> index 62a6ba3..0a6a944 100644
> --- a/arch/mips/include/asm/pgtable.h
> +++ b/arch/mips/include/asm/pgtable.h
> @@ -148,6 +148,7 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
> }
> }
> }
> +#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
>
> static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
> {
> @@ -185,6 +186,7 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
> }
> #endif
> }
> +#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
>
> static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
> {
> @@ -401,12 +403,15 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
>
> extern void __update_tlb(struct vm_area_struct *vma, unsigned long address,
> pte_t pte);
> +extern void __update_cache(struct vm_area_struct *vma, unsigned long address,
> + pte_t pte);
>
> static inline void update_mmu_cache(struct vm_area_struct *vma,
> unsigned long address, pte_t *ptep)
> {
> pte_t pte = *ptep;
> __update_tlb(vma, address, pte);
> + __update_cache(vma, address, pte);
> }
>
> static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
> diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
> index dd261df..e045116 100644
> --- a/arch/mips/mm/c-r4k.c
> +++ b/arch/mips/mm/c-r4k.c
> @@ -484,8 +484,11 @@ static inline void local_r4k_flush_cache_range(void * args)
> return;
>
> r4k_blast_dcache();
> - if (exec)
> + if (exec) {
> + if (!cpu_has_ic_fills_f_dc)
> + wmb();
> r4k_blast_icache();
> + }
> }
>
> static void r4k_flush_cache_range(struct vm_area_struct *vma,
> @@ -570,6 +573,14 @@ static inline void local_r4k_flush_cache_page(void *args)
> if (!(pte_present(*ptep)))
> return;
>
> + /*
> + * If this page is not destined to be executable and the
> + * data cache does not have aliases, all of the mapping
> + * below can be skipped.
> + */
> + if (!exec && !cpu_has_dc_aliases)
> + return;
> +
> if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
> vaddr = NULL;
> else {
> @@ -589,6 +600,8 @@ static inline void local_r4k_flush_cache_page(void *args)
> if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
> vaddr ? r4k_blast_dcache_page(addr) :
> r4k_blast_dcache_user_page(addr);
> + if (exec && !cpu_has_ic_fills_f_dc)
> + wmb();
> if (exec && !cpu_icache_snoops_remote_store)
> r4k_blast_scache_page(addr);
> }
> @@ -621,6 +634,8 @@ static void r4k_flush_cache_page(struct vm_area_struct *vma,
> args.pfn = pfn;
>
> r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
> + if (cpu_has_dc_aliases)
> + ClearPageDcacheDirty(pfn_to_page(pfn));
> }
>
> static inline void local_r4k_flush_data_cache_page(void * addr)
> @@ -652,6 +667,8 @@ static inline void local_r4k_flush_icache_range(unsigned long start, unsigned lo
> }
> }
>
> + wmb();
> +
> if (end - start > icache_size)
> r4k_blast_icache();
> else {
> diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
> index 7e3ea77..99db9e8 100644
> --- a/arch/mips/mm/cache.c
> +++ b/arch/mips/mm/cache.c
> @@ -19,6 +19,7 @@
> #include <asm/processor.h>
> #include <asm/cpu.h>
> #include <asm/cpu-features.h>
> +#include <linux/highmem.h>
>
> /* Cache operations. */
> void (*flush_cache_all)(void);
> @@ -105,48 +106,47 @@ void __flush_anon_page(struct page *page, unsigned long vmaddr)
> {
> unsigned long addr = (unsigned long) page_address(page);
>
> - if (pages_do_alias(addr, vmaddr)) {
> + if (pages_do_alias(addr, vmaddr & PAGE_MASK)) {
> if (page_mapped(page) && !Page_dcache_dirty(page)) {
> void *kaddr;
>
> kaddr = kmap_coherent(page, vmaddr);
> flush_data_cache_page((unsigned long)kaddr);
> kunmap_coherent();
> - } else
> - flush_data_cache_page(addr);
> + } else {
> + void *kaddr;
> +
> + kaddr = kmap_atomic(page);
> + flush_data_cache_page((unsigned long)kaddr);
> + kunmap_atomic(kaddr);
> + ClearPageDcacheDirty(page);
> + }
> }
> }
>
> EXPORT_SYMBOL(__flush_anon_page);
>
> -static void mips_flush_dcache_from_pte(pte_t pteval, unsigned long address)
> +void __update_cache(struct vm_area_struct *vma, unsigned long address,
> + pte_t pte)
> {
> struct page *page;
> - unsigned long pfn = pte_pfn(pteval);
> + unsigned long pfn = pte_pfn(pte);
> + int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc;
>
> - if (unlikely(!pfn_valid(pfn)))
> + if (unlikely(!pfn_valid(pfn))) {
> + wmb();
> return;
> -
> + }
> page = pfn_to_page(pfn);
> - if (page_mapping(page) && Page_dcache_dirty(page)) {
> + if (page_mapped(page) && Page_dcache_dirty(page)) {
> unsigned long page_addr = (unsigned long) page_address(page);
> -
> - if (!cpu_has_ic_fills_f_dc ||
> - pages_do_alias(page_addr, address & PAGE_MASK))
> + if (exec || (cpu_has_dc_aliases &&
> + pages_do_alias(page_addr, address & PAGE_MASK))) {
> flush_data_cache_page(page_addr);
> - ClearPageDcacheDirty(page);
> + ClearPageDcacheDirty(page);
> + }
> }
> -}
> -
> -void set_pte_at(struct mm_struct *mm, unsigned long addr,
> - pte_t *ptep, pte_t pteval)
> -{
> - if (cpu_has_dc_aliases || !cpu_has_ic_fills_f_dc) {
> - if (pte_present(pteval))
> - mips_flush_dcache_from_pte(pteval, addr);
> - }
> -
> - set_pte(ptep, pteval);
> + wmb(); /* finish any outstanding arch cache flushes before ret to user */
> }
>
> unsigned long _page_cachable_default;
> diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
> index 448cde3..597bf7f 100644
> --- a/arch/mips/mm/init.c
> +++ b/arch/mips/mm/init.c
> @@ -165,9 +165,15 @@ void copy_user_highpage(struct page *to, struct page *from,
> copy_page(vto, vfrom);
> kunmap_atomic(vfrom);
> }
> - if ((!cpu_has_ic_fills_f_dc) ||
> - pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
> + if (cpu_has_dc_aliases)
> + SetPageDcacheDirty(to);
> + if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
> + cpu_has_vtag_dcache || (cpu_has_dc_aliases &&
> + pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))) {
> flush_data_cache_page((unsigned long)vto);
> + if (cpu_has_dc_aliases)
> + ClearPageDcacheDirty(to);
> + }
> kunmap_atomic(vto);
> /* Make sure this page is cleared on other CPU's too before using it */
> smp_wmb();
> @@ -187,8 +193,14 @@ void copy_to_user_page(struct vm_area_struct *vma,
> if (cpu_has_dc_aliases)
> SetPageDcacheDirty(page);
> }
> - if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
> + if (((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) ||
> + (Page_dcache_dirty(page) &&
> + pages_do_alias((unsigned long)dst & PAGE_MASK,
> + vaddr & PAGE_MASK))) {
> flush_cache_page(vma, vaddr, page_to_pfn(page));
> + if (cpu_has_dc_aliases)
> + ClearPageDcacheDirty(page);
> + }
> }
>
> void copy_from_user_page(struct vm_area_struct *vma,
> @@ -200,11 +212,8 @@ void copy_from_user_page(struct vm_area_struct *vma,
> void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
> memcpy(dst, vfrom, len);
> kunmap_coherent();
> - } else {
> + } else
> memcpy(dst, src, len);
> - if (cpu_has_dc_aliases)
> - SetPageDcacheDirty(page);
> - }
> }
> EXPORT_SYMBOL_GPL(copy_from_user_page);
>
> diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
> index 99eb8fa..e1db410 100644
> --- a/arch/mips/mm/sc-mips.c
> +++ b/arch/mips/mm/sc-mips.c
> @@ -24,6 +24,7 @@
> */
> static void mips_sc_wback_inv(unsigned long addr, unsigned long size)
> {
> + __sync();
> blast_scache_range(addr, addr + size);
> }
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 2/3] MIPS: Highmem: Fixes for cache aliasing and color.
2015-01-29 16:46 [PATCH 0/3] HIGHMEM and cache flush fixes Steven J. Hill
2015-01-29 16:46 ` [PATCH 1/3] MIPS: Fix cache flushing for swap pages with non-DMA I/O Steven J. Hill
@ 2015-01-29 16:46 ` Steven J. Hill
2015-01-29 16:46 ` [PATCH 3/3] MIPS: Fix I-cache flushing for kmap'd pages Steven J. Hill
2 siblings, 0 replies; 6+ messages in thread
From: Steven J. Hill @ 2015-01-29 16:46 UTC (permalink / raw)
To: linux-mips; +Cc: ralf
From: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
Make HIGHMEM kmap cache coloring aware and fix some corner
cases for cache aliasing.
Signed-off-by: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
Signed-off-by: Steven J. Hill <Steven.Hill@imgtec.com>
---
arch/mips/Kconfig | 1 +
arch/mips/include/asm/fixmap.h | 18 ++++++++-
arch/mips/include/asm/highmem.h | 33 +++++++++++++++
arch/mips/mm/c-r4k.c | 8 ++++
arch/mips/mm/cache.c | 84 ++++++++++++++++++++++++++-------------
arch/mips/mm/highmem.c | 43 ++++++++------------
arch/mips/mm/init.c | 10 +----
7 files changed, 132 insertions(+), 65 deletions(-)
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 3289969..1fae6f1 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -391,6 +391,7 @@ config MIPS_MALTA
select SYS_SUPPORTS_MULTITHREADING
select SYS_SUPPORTS_SMARTMIPS
select SYS_SUPPORTS_ZBOOT
+ select SYS_SUPPORTS_HIGHMEM
help
This enables support for the MIPS Technologies Malta evaluation
board.
diff --git a/arch/mips/include/asm/fixmap.h b/arch/mips/include/asm/fixmap.h
index 6842ffa..f7733a0 100644
--- a/arch/mips/include/asm/fixmap.h
+++ b/arch/mips/include/asm/fixmap.h
@@ -46,13 +46,27 @@
* fix-mapped?
*/
enum fixed_addresses {
+
+/*
+ * Must be <=8 but, last_pkmap_nr_arr[] is still initialized to
+ * 8 elements. Keep the total L1 size <= 512KB with 4-ways.
+ */
+#ifdef CONFIG_PAGE_SIZE_64KB
+#define FIX_N_COLOURS 2
+#endif
+#ifdef CONFIG_PAGE_SIZE_32KB
+#define FIX_N_COLOURS 4
+#endif
+#ifndef FIX_N_COLOURS
#define FIX_N_COLOURS 8
+#endif
+
FIX_CMAP_BEGIN,
- FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * 2),
+ FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * NR_CPUS * 2),
#ifdef CONFIG_HIGHMEM
/* reserved pte's for temporary kernel mappings */
FIX_KMAP_BEGIN = FIX_CMAP_END + 1,
- FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
+ FIX_KMAP_END = FIX_KMAP_BEGIN+(8*NR_CPUS*FIX_N_COLOURS)-1,
#endif
__end_of_fixed_addresses
};
diff --git a/arch/mips/include/asm/highmem.h b/arch/mips/include/asm/highmem.h
index 572e63e..af6180f 100644
--- a/arch/mips/include/asm/highmem.h
+++ b/arch/mips/include/asm/highmem.h
@@ -36,11 +36,44 @@ extern pte_t *pkmap_page_table;
* easily, subsequent pte tables have to be allocated in one physical
* chunk of RAM.
*/
+
+/* 8 colors pages are here */
+#ifdef CONFIG_PAGE_SIZE_4KB
+#define LAST_PKMAP 4096
+#endif
+#ifdef CONFIG_PAGE_SIZE_8KB
+#define LAST_PKMAP 2048
+#endif
+#ifdef CONFIG_PAGE_SIZE_16KB
+#define LAST_PKMAP 1024
+#endif
+
+/* 32KB and 64KB pages should have 4 and 2 colors to keep space under control */
+#ifndef LAST_PKMAP
#define LAST_PKMAP 1024
+#endif
+
#define LAST_PKMAP_MASK (LAST_PKMAP-1)
#define PKMAP_NR(virt) ((virt-PKMAP_BASE) >> PAGE_SHIFT)
#define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT))
+#define get_pkmap_color(pg) (((unsigned long)lowmem_page_address(pg) >> PAGE_SHIFT) & (FIX_N_COLOURS-1))
+#define get_last_pkmap_nr(cl) (last_pkmap_nr_arr[cl])
+#define get_next_pkmap_nr(cl) (last_pkmap_nr_arr[cl] = \
+ ((get_last_pkmap_nr(cl) + FIX_N_COLOURS) & LAST_PKMAP_MASK))
+#define no_more_pkmaps(p,cl) (p < FIX_N_COLOURS)
+#define get_pkmap_entries_count(c) (c - FIX_N_COLOURS)
+
+static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color)
+{
+ static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
+
+ return &pkmap_map_wait;
+}
+
+extern unsigned int last_pkmap_nr_arr[];
+
+
extern void * kmap_high(struct page *page);
extern void kunmap_high(struct page *page);
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index e045116..9096c5f 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -1288,6 +1288,14 @@ static void probe_pcache(void)
c->dcache.flags |= MIPS_CACHE_ALIASES;
}
+#ifdef CONFIG_HIGHMEM
+ if (((c->dcache.flags & MIPS_CACHE_ALIASES) &&
+ ((c->dcache.waysize / PAGE_SIZE) > FIX_N_COLOURS)) ||
+ ((c->icache.flags & MIPS_CACHE_ALIASES) &&
+ ((c->icache.waysize / PAGE_SIZE) > FIX_N_COLOURS)))
+ panic("PAGE_SIZE*WAYS too small for L1 size, too many colors");
+#endif
+
switch (current_cpu_type()) {
case CPU_20KC:
/*
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
index 99db9e8..49496a4 100644
--- a/arch/mips/mm/cache.c
+++ b/arch/mips/mm/cache.c
@@ -81,12 +81,9 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
void __flush_dcache_page(struct page *page)
{
- struct address_space *mapping = page_mapping(page);
- unsigned long addr;
+ void *addr;
- if (PageHighMem(page))
- return;
- if (mapping && !mapping_mapped(mapping)) {
+ if (page_mapping(page) && !page_mapped(page)) {
SetPageDcacheDirty(page);
return;
}
@@ -96,30 +93,54 @@ void __flush_dcache_page(struct page *page)
* case is for exec env/arg pages and those are %99 certainly going to
* get faulted into the tlb (and thus flushed) anyways.
*/
- addr = (unsigned long) page_address(page);
- flush_data_cache_page(addr);
+ if (PageHighMem(page)) {
+ addr = kmap_atomic(page);
+ flush_data_cache_page((unsigned long)addr);
+ kunmap_atomic(addr);
+ } else {
+ addr = (void *) page_address(page);
+ flush_data_cache_page((unsigned long)addr);
+ }
+ ClearPageDcacheDirty(page);
}
EXPORT_SYMBOL(__flush_dcache_page);
void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
- unsigned long addr = (unsigned long) page_address(page);
-
- if (pages_do_alias(addr, vmaddr & PAGE_MASK)) {
- if (page_mapped(page) && !Page_dcache_dirty(page)) {
- void *kaddr;
-
- kaddr = kmap_coherent(page, vmaddr);
- flush_data_cache_page((unsigned long)kaddr);
- kunmap_coherent();
- } else {
- void *kaddr;
-
- kaddr = kmap_atomic(page);
- flush_data_cache_page((unsigned long)kaddr);
- kunmap_atomic(kaddr);
- ClearPageDcacheDirty(page);
+ if (!PageHighMem(page)) {
+ unsigned long addr = (unsigned long) page_address(page);
+
+ if (pages_do_alias(addr, vmaddr & PAGE_MASK)) {
+ if (page_mapped(page) && !Page_dcache_dirty(page)) {
+ void *kaddr;
+
+ kaddr = kmap_coherent(page, vmaddr);
+ flush_data_cache_page((unsigned long)kaddr);
+ kunmap_coherent();
+ } else {
+ flush_data_cache_page(addr);
+ ClearPageDcacheDirty(page);
+ }
+ }
+ } else {
+ void *laddr = lowmem_page_address(page);
+
+ if (pages_do_alias((unsigned long)laddr, vmaddr & PAGE_MASK)) {
+ if (page_mapped(page) && !Page_dcache_dirty(page)) {
+ void *kaddr;
+
+ kaddr = kmap_coherent(page, vmaddr);
+ flush_data_cache_page((unsigned long)kaddr);
+ kunmap_coherent();
+ } else {
+ void *kaddr;
+
+ kaddr = kmap_atomic(page);
+ flush_data_cache_page((unsigned long)kaddr);
+ kunmap_atomic(kaddr);
+ ClearPageDcacheDirty(page);
+ }
}
}
}
@@ -130,21 +151,30 @@ void __update_cache(struct vm_area_struct *vma, unsigned long address,
pte_t pte)
{
struct page *page;
- unsigned long pfn = pte_pfn(pte);
+ unsigned long pfn, addr;
int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc;
+ pfn = pte_pfn(pte);
if (unlikely(!pfn_valid(pfn))) {
wmb();
return;
}
page = pfn_to_page(pfn);
if (page_mapped(page) && Page_dcache_dirty(page)) {
- unsigned long page_addr = (unsigned long) page_address(page);
+ void *kaddr = NULL;
+ if (PageHighMem(page)) {
+ addr = (unsigned long)kmap_atomic(page);
+ kaddr = (void *)addr;
+ } else
+ addr = (unsigned long) page_address(page);
if (exec || (cpu_has_dc_aliases &&
- pages_do_alias(page_addr, address & PAGE_MASK))) {
- flush_data_cache_page(page_addr);
+ pages_do_alias(addr, address & PAGE_MASK))) {
+ flush_data_cache_page(addr);
ClearPageDcacheDirty(page);
}
+
+ if (kaddr)
+ kunmap_atomic((void *)kaddr);
}
wmb(); /* finish any outstanding arch cache flushes before ret to user */
}
diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
index da815d2..10fc041 100644
--- a/arch/mips/mm/highmem.c
+++ b/arch/mips/mm/highmem.c
@@ -9,6 +9,7 @@
static pte_t *kmap_pte;
unsigned long highstart_pfn, highend_pfn;
+unsigned int last_pkmap_nr_arr[FIX_N_COLOURS] = { 0, 1, 2, 3, 4, 5, 6, 7 };
void *kmap(struct page *page)
{
@@ -53,8 +54,12 @@ void *kmap_atomic(struct page *page)
return page_address(page);
type = kmap_atomic_idx_push();
- idx = type + KM_TYPE_NR*smp_processor_id();
- vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+
+ idx = (((unsigned long)lowmem_page_address(page)) >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
+ idx = (FIX_N_COLOURS - idx);
+ idx = idx + FIX_N_COLOURS * (smp_processor_id() + NR_CPUS * type);
+ vaddr = __fix_to_virt(FIX_KMAP_BEGIN - 1 + idx); /* actually - FIX_CMAP_END */
+
#ifdef CONFIG_DEBUG_HIGHMEM
BUG_ON(!pte_none(*(kmap_pte - idx)));
#endif
@@ -75,12 +80,16 @@ void __kunmap_atomic(void *kvaddr)
return;
}
- type = kmap_atomic_idx();
#ifdef CONFIG_DEBUG_HIGHMEM
{
- int idx = type + KM_TYPE_NR * smp_processor_id();
+ int idx;
+ type = kmap_atomic_idx();
- BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
+ idx = ((unsigned long)kvaddr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
+ idx = (FIX_N_COLOURS - idx);
+ idx = idx + FIX_N_COLOURS * (smp_processor_id() + NR_CPUS * type);
+
+ BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN -1 + idx));
/*
* force other mappings to Oops if they'll try to access
@@ -95,26 +104,6 @@ void __kunmap_atomic(void *kvaddr)
}
EXPORT_SYMBOL(__kunmap_atomic);
-/*
- * This is the same as kmap_atomic() but can map memory that doesn't
- * have a struct page associated with it.
- */
-void *kmap_atomic_pfn(unsigned long pfn)
-{
- unsigned long vaddr;
- int idx, type;
-
- pagefault_disable();
-
- type = kmap_atomic_idx_push();
- idx = type + KM_TYPE_NR*smp_processor_id();
- vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
- set_pte(kmap_pte-idx, pfn_pte(pfn, PAGE_KERNEL));
- flush_tlb_one(vaddr);
-
- return (void*) vaddr;
-}
-
struct page *kmap_atomic_to_page(void *ptr)
{
unsigned long idx, vaddr = (unsigned long)ptr;
@@ -124,7 +113,7 @@ struct page *kmap_atomic_to_page(void *ptr)
return virt_to_page(ptr);
idx = virt_to_fix(vaddr);
- pte = kmap_pte - (idx - FIX_KMAP_BEGIN);
+ pte = kmap_pte - (idx - FIX_KMAP_BEGIN + 1);
return pte_page(*pte);
}
@@ -133,6 +122,6 @@ void __init kmap_init(void)
unsigned long kmap_vstart;
/* cache the first kmap pte */
- kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
+ kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN - 1); /* actually - FIX_CMAP_END */
kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
}
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 597bf7f..0dc604a 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -284,7 +284,7 @@ int page_is_ram(unsigned long pagenr)
void __init paging_init(void)
{
unsigned long max_zone_pfns[MAX_NR_ZONES];
- unsigned long lastpfn __maybe_unused;
+ unsigned long lastpfn;
pagetable_init();
@@ -302,14 +302,6 @@ void __init paging_init(void)
#ifdef CONFIG_HIGHMEM
max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
lastpfn = highend_pfn;
-
- if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
- printk(KERN_WARNING "This processor doesn't support highmem."
- " %ldk highmem ignored\n",
- (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
- max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
- lastpfn = max_low_pfn;
- }
#endif
free_area_init_nodes(max_zone_pfns);
--
1.7.10.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/3] MIPS: Fix I-cache flushing for kmap'd pages.
2015-01-29 16:46 [PATCH 0/3] HIGHMEM and cache flush fixes Steven J. Hill
2015-01-29 16:46 ` [PATCH 1/3] MIPS: Fix cache flushing for swap pages with non-DMA I/O Steven J. Hill
2015-01-29 16:46 ` [PATCH 2/3] MIPS: Highmem: Fixes for cache aliasing and color Steven J. Hill
@ 2015-01-29 16:46 ` Steven J. Hill
2 siblings, 0 replies; 6+ messages in thread
From: Steven J. Hill @ 2015-01-29 16:46 UTC (permalink / raw)
To: linux-mips; +Cc: ralf
From: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
Make the I-cache flush pages while taking into account the
address color by using kmap_coherent() when there is
I-cache aliasing present.
Signed-off-by: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com>
Signed-off-by: Steven J. Hill <Steven.Hill@imgtec.com>
---
arch/mips/include/asm/cpu-features.h | 3 +++
arch/mips/mm/c-r4k.c | 17 ++++++++++++++---
arch/mips/mm/init.c | 2 --
3 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
index 23db770..92aa321 100644
--- a/arch/mips/include/asm/cpu-features.h
+++ b/arch/mips/include/asm/cpu-features.h
@@ -142,6 +142,9 @@
#ifndef cpu_has_vtag_dcache
#define cpu_has_vtag_dcache (cpu_data[0].dcache.flags & MIPS_CACHE_VTAG)
#endif
+#ifndef cpu_has_ic_aliases
+#define cpu_has_ic_aliases (cpu_data[0].icache.flags & MIPS_CACHE_ALIASES)
+#endif
#ifndef cpu_has_dc_aliases
#define cpu_has_dc_aliases (cpu_data[0].dcache.flags & MIPS_CACHE_ALIASES)
#endif
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 9096c5f..d48da56a 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -552,6 +552,7 @@ static inline void local_r4k_flush_cache_page(void *args)
pmd_t *pmdp;
pte_t *ptep;
void *vaddr;
+ int noflush = 0;
/*
* If ownes no valid ASID yet, cannot possibly have gotten
@@ -611,6 +612,7 @@ static inline void local_r4k_flush_cache_page(void *args)
if (cpu_context(cpu, mm) != 0)
drop_mmu_context(mm, cpu);
+ noflush = 1;
} else
vaddr ? r4k_blast_icache_page(addr) :
r4k_blast_icache_user_page(addr);
@@ -622,6 +624,13 @@ static inline void local_r4k_flush_cache_page(void *args)
else
kunmap_atomic(vaddr);
}
+
+ /* If we have I-cache aliasing, then blast it via coherent page. */
+ if (exec && cpu_has_ic_aliases && !noflush && !map_coherent) {
+ vaddr = kmap_coherent(page, addr);
+ r4k_blast_icache_page((unsigned long)vaddr);
+ kunmap_coherent();
+ }
}
static void r4k_flush_cache_page(struct vm_area_struct *vma,
@@ -1317,10 +1326,12 @@ static void probe_pcache(void)
c->icache.ways = 1;
}
- printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
- icache_size >> 10,
+ printk("Primary instruction cache %ldkB, %s, %s, %slinesize %d bytes.\n",
+ icache_size >> 10, way_string[c->icache.ways],
c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
- way_string[c->icache.ways], c->icache.linesz);
+ (c->icache.flags & MIPS_CACHE_ALIASES) ?
+ "I-cache aliases, " : "",
+ c->icache.linesz);
printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
dcache_size >> 10, way_string[c->dcache.ways],
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 0dc604a..b1584d6 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -88,8 +88,6 @@ static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
pte_t pte;
int tlbidx;
- BUG_ON(Page_dcache_dirty(page));
-
pagefault_disable();
idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
idx += in_interrupt() ? FIX_N_COLOURS : 0;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 6+ messages in thread