* [RESEND PATCH v4 0/6] arm64/mm: Move
@ 2018-07-18 10:17 Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 1/6] arm64/mm: Introduce init_pg_dir Jun Yao
` (5 more replies)
0 siblings, 6 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
Version 4 changes:
1. Rename INIT_DIR to INIT_PG_TABLES and move it outside the
'.init.data' section[1].
2. Move the 'count' calculation to clear_pages()[1].
3. Rearrange register in __enable_mmu() and pass ttbr1 through
the x1 register[2].
4. Add commit message to explain why we can make swapper_pg_dir
smaller[3].
5. Rewrite in_swapper_pgdir() and use pgd_set_fixmap() to
populate swapper_pg_dir. At the same time, add a spin lock to
avoid race conditions[4].
6. Move {idmap_pg_dir,tramp_pg_dir,reserved_ttbr0,swapper_pg_dir}
to the KERNEL_PG_TABLES macro. And put this macro after
NOTES[4].
7. Update init_mm.pgd before kasan_early_init() and
kaslr_early_init(), which makes pgd_offset_k() work properly.
Test following configs with CONFIG_RANDOMIZE_BASE/UNMAP_KERNEL_AT_EL0/
CONFIG_ARM64_SW_TTBR0_PAN enabled on qemu:
1. CONFIG_ARM64_4K_PAGES/CONFIG_ARM64_VA_BITS_48
2. CONFIG_ARM64_4K_PAGES/CONFIG_ARM64_VA_BITS_39
3. CONFIG_ARM64_64K_PAGES/CONFIG_ARM64_VA_BITS_48
4. CONFIG_ARM64_64K_PAGES/CONFIG_ARM64_VA_BITS_42
v3: https://www.spinics.net/lists/arm-kernel/msg662537.html
v2: https://patchwork.kernel.org/patch/10485641/
v1: https://patchwork.kernel.org/patch/10476595/
[1] https://lkml.org/lkml/2018/7/6/238
[2] https://lkml.org/lkml/2018/7/6/239
[3] https://lkml.org/lkml/2018/7/6/243
[4] https://lkml.org/lkml/2018/7/11/782
Jun Yao (6):
arm64/mm: Introduce init_pg_dir
arm64/mm: Make __enable_mmu() take the ttbr1 page as an argument
arm64/mm: Create initial page tables in init_pg_dir
arm64/mm: Make swapper_pg_dir smaller
arm64/mm: Populate swapper_pg_dir by fixmap
arm64/mm: Move {idmap_pg_dir, swapper_pg_dir} to .rodata section
arch/arm64/include/asm/assembler.h | 29 +++++++++++++
arch/arm64/include/asm/pgtable.h | 66 ++++++++++++++++++++++++++----
arch/arm64/kernel/head.S | 48 ++++++++++++++--------
arch/arm64/kernel/sleep.S | 1 +
arch/arm64/kernel/vmlinux.lds.S | 47 ++++++++++++++-------
arch/arm64/mm/mmu.c | 35 ++++------------
6 files changed, 159 insertions(+), 67 deletions(-)
--
2.17.1
^ permalink raw reply [flat|nested] 7+ messages in thread
* [RESEND PATCH v4 1/6] arm64/mm: Introduce init_pg_dir
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
@ 2018-07-18 10:17 ` Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 2/6] arm64/mm: Make __enable_mmu() take the Jun Yao
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
Add init_pg_dir to vmlinux.lds.S and boiler-plate
clearing/cleaning/invalidating it in head.S.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
---
arch/arm64/include/asm/assembler.h | 29 +++++++++++++++++++++++++++++
arch/arm64/kernel/head.S | 22 +++++++++++++++-------
arch/arm64/kernel/vmlinux.lds.S | 8 ++++++++
3 files changed, 52 insertions(+), 7 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 0bcc98dbba56..eb363a915c0e 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -456,6 +456,35 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
b.ne 9998b
.endm
+/*
+ * clear_page - clear one page
+ *
+ * start: page aligned virtual address
+ */
+ .macro clear_page, start:req
+9996: stp xzr, xzr, [\start], #16
+ stp xzr, xzr, [\start], #16
+ stp xzr, xzr, [\start], #16
+ stp xzr, xzr, [\start], #16
+ tst \start, #(PAGE_SIZE - 1)
+ b.ne 9996b
+ .endm
+
+/*
+ * clear_pages - clear contiguous pages
+ *
+ * start, end: page aligend virtual addresses
+ */
+ .macro clear_pages, start:req, end:req
+ sub \end, \end, \start
+ lsr \end, \end, #(PAGE_SHIFT)
+9997: cbz \end, 9998f
+ clear_page \start
+ sub \end, \end, #1
+ b 9997b
+9998:
+ .endm
+
/*
* Annotate a function as position independent, i.e., safe to be called before
* the kernel virtual mapping is activated.
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index b0853069702f..2c83a8c47e3f 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -295,18 +295,21 @@ __create_page_tables:
sub x1, x1, x0
bl __inval_dcache_area
+ adrp x0, init_pg_dir
+ adrp x1, init_pg_end
+ sub x1, x1, x0
+ bl __inval_dcache_area
+
/*
* Clear the idmap and swapper page tables.
*/
adrp x0, idmap_pg_dir
adrp x1, swapper_pg_end
- sub x1, x1, x0
-1: stp xzr, xzr, [x0], #16
- stp xzr, xzr, [x0], #16
- stp xzr, xzr, [x0], #16
- stp xzr, xzr, [x0], #16
- subs x1, x1, #64
- b.ne 1b
+ clear_pages x0, x1
+
+ adrp x0, init_pg_dir
+ adrp x1, init_pg_end
+ clear_pages x0, x1
mov x7, SWAPPER_MM_MMUFLAGS
@@ -395,6 +398,11 @@ __create_page_tables:
dmb sy
bl __inval_dcache_area
+ adrp x0, init_pg_dir
+ adrp x1, init_pg_end
+ sub x1, x1, x0
+ bl __inval_dcache_area
+
ret x28
ENDPROC(__create_page_tables)
.ltorg
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 605d1b60469c..61d7cee3eaa6 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -68,6 +68,12 @@ jiffies = jiffies_64;
#define TRAMP_TEXT
#endif
+#define INIT_PG_TABLES \
+ . = ALIGN(PAGE_SIZE); \
+ init_pg_dir = .; \
+ . += SWAPPER_DIR_SIZE; \
+ init_pg_end = .;
+
/*
* The size of the PE/COFF section that covers the kernel image, which
* runs from stext to _edata, must be a round multiple of the PE/COFF
@@ -161,6 +167,8 @@ SECTIONS
__inittext_end = .;
__initdata_begin = .;
+ INIT_PG_TABLES
+
.init.data : {
INIT_DATA
INIT_SETUP(16)
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [RESEND PATCH v4 2/6] arm64/mm: Make __enable_mmu() take the
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 1/6] arm64/mm: Introduce init_pg_dir Jun Yao
@ 2018-07-18 10:17 ` Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 3/6] arm64/mm: Create initial page tables in Jun Yao
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
Make __enable_mmu() take the physical address of the ttbr1 page as
an argument.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
---
arch/arm64/kernel/head.S | 21 ++++++++++++---------
arch/arm64/kernel/sleep.S | 1 +
2 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 2c83a8c47e3f..c3e4b1886cde 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -714,6 +714,7 @@ secondary_startup:
* Common entry point for secondary CPUs.
*/
bl __cpu_setup // initialise processor
+ adrp x1, swapper_pg_dir
bl __enable_mmu
ldr x8, =__secondary_switched
br x8
@@ -756,6 +757,7 @@ ENDPROC(__secondary_switched)
* Enable the MMU.
*
* x0 = SCTLR_EL1 value for turning on the MMU.
+ * x1 = TTBR1_EL1 value for turning on the MMU.
*
* Returns to the caller via x30/lr. This requires the caller to be covered
* by the .idmap.text section.
@@ -764,15 +766,15 @@ ENDPROC(__secondary_switched)
* If it isn't, park the CPU
*/
ENTRY(__enable_mmu)
- mrs x1, ID_AA64MMFR0_EL1
- ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4
- cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED
+ mrs x5, ID_AA64MMFR0_EL1
+ ubfx x6, x5, #ID_AA64MMFR0_TGRAN_SHIFT, 4
+ cmp x6, #ID_AA64MMFR0_TGRAN_SUPPORTED
b.ne __no_granule_support
- update_early_cpu_boot_status 0, x1, x2
- adrp x1, idmap_pg_dir
- adrp x2, swapper_pg_dir
- phys_to_ttbr x3, x1
- phys_to_ttbr x4, x2
+ update_early_cpu_boot_status 0, x5, x6
+ adrp x5, idmap_pg_dir
+ mov x6, x1
+ phys_to_ttbr x3, x5
+ phys_to_ttbr x4, x6
msr ttbr0_el1, x3 // load TTBR0
msr ttbr1_el1, x4 // load TTBR1
isb
@@ -791,7 +793,7 @@ ENDPROC(__enable_mmu)
__no_granule_support:
/* Indicate that this CPU can't boot and is stuck in the kernel */
- update_early_cpu_boot_status CPU_STUCK_IN_KERNEL, x1, x2
+ update_early_cpu_boot_status CPU_STUCK_IN_KERNEL, x5, x6
1:
wfe
wfi
@@ -831,6 +833,7 @@ __primary_switch:
mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value
#endif
+ adrp x1, swapper_pg_dir
bl __enable_mmu
#ifdef CONFIG_RELOCATABLE
bl __relocate_kernel
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index bebec8ef9372..3e53ffa07994 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -101,6 +101,7 @@ ENTRY(cpu_resume)
bl el2_setup // if in EL2 drop to EL1 cleanly
bl __cpu_setup
/* enable the MMU early - so we can access sleep_save_stash by va */
+ adrp x1, swapper_pg_dir
bl __enable_mmu
ldr x8, =_cpu_resume
br x8
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [RESEND PATCH v4 3/6] arm64/mm: Create initial page tables in
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 1/6] arm64/mm: Introduce init_pg_dir Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 2/6] arm64/mm: Make __enable_mmu() take the Jun Yao
@ 2018-07-18 10:17 ` Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 4/6] arm64/mm: Make swapper_pg_dir smaller Jun Yao
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
Create initial page tables in init_pg_dir and then create final
page tables in swapper_pg_dir directly.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
---
arch/arm64/include/asm/pgtable.h | 2 ++
arch/arm64/kernel/head.S | 9 ++++++---
arch/arm64/mm/mmu.c | 27 ++++++++-------------------
3 files changed, 16 insertions(+), 22 deletions(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 1bdeca8918a6..46ef21ebfe47 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -712,6 +712,8 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
}
#endif
+extern pgd_t init_pg_dir[PTRS_PER_PGD];
+extern pgd_t init_pg_end[];
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
extern pgd_t swapper_pg_end[];
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index c3e4b1886cde..ede2e964592b 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -376,7 +376,7 @@ __create_page_tables:
/*
* Map the kernel image (starting with PHYS_OFFSET).
*/
- adrp x0, swapper_pg_dir
+ adrp x0, init_pg_dir
mov_q x5, KIMAGE_VADDR + TEXT_OFFSET // compile time __va(_text)
add x5, x5, x23 // add KASLR displacement
mov x4, PTRS_PER_PGD
@@ -402,7 +402,6 @@ __create_page_tables:
adrp x1, init_pg_end
sub x1, x1, x0
bl __inval_dcache_area
-
ret x28
ENDPROC(__create_page_tables)
.ltorg
@@ -439,6 +438,9 @@ __primary_switched:
bl __pi_memset
dsb ishst // Make zero page visible to PTW
+ adrp x0, init_pg_dir
+ bl set_init_mm_pgd
+
#ifdef CONFIG_KASAN
bl kasan_early_init
#endif
@@ -833,8 +835,9 @@ __primary_switch:
mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value
#endif
- adrp x1, swapper_pg_dir
+ adrp x1, init_pg_dir
bl __enable_mmu
+
#ifdef CONFIG_RELOCATABLE
bl __relocate_kernel
#ifdef CONFIG_RANDOMIZE_BASE
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 65f86271f02b..088a591e4ea4 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -629,26 +629,10 @@ static void __init map_kernel(pgd_t *pgdp)
*/
void __init paging_init(void)
{
- phys_addr_t pgd_phys = early_pgtable_alloc();
- pgd_t *pgdp = pgd_set_fixmap(pgd_phys);
-
- map_kernel(pgdp);
- map_mem(pgdp);
-
- /*
- * We want to reuse the original swapper_pg_dir so we don't have to
- * communicate the new address to non-coherent secondaries in
- * secondary_entry, and so cpu_switch_mm can generate the address with
- * adrp+add rather than a load from some global variable.
- *
- * To do this we need to go via a temporary pgd.
- */
- cpu_replace_ttbr1(__va(pgd_phys));
- memcpy(swapper_pg_dir, pgdp, PGD_SIZE);
+ map_kernel(swapper_pg_dir);
+ map_mem(swapper_pg_dir);
cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
-
- pgd_clear_fixmap();
- memblock_free(pgd_phys, PAGE_SIZE);
+ init_mm.pgd = swapper_pg_dir;
/*
* We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
@@ -659,6 +643,11 @@ void __init paging_init(void)
- PAGE_SIZE);
}
+void __init set_init_mm_pgd(pgd_t *pgd)
+{
+ init_mm.pgd = pgd;
+}
+
/*
* Check whether a kernel address is valid (derived from arch/x86/).
*/
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [RESEND PATCH v4 4/6] arm64/mm: Make swapper_pg_dir smaller
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
` (2 preceding siblings ...)
2018-07-18 10:17 ` [RESEND PATCH v4 3/6] arm64/mm: Create initial page tables in Jun Yao
@ 2018-07-18 10:17 ` Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 5/6] arm64/mm: Populate swapper_pg_dir by Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 6/6] arm64/mm: Move Jun Yao
5 siblings, 0 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
We setup final page tables in swapper_pg_dir, which just contains
PGD. And the PUD/PMD are allocated dynamically. So we can make it
smaller.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
---
arch/arm64/kernel/vmlinux.lds.S | 2 +-
arch/arm64/mm/mmu.c | 8 --------
2 files changed, 1 insertion(+), 9 deletions(-)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 61d7cee3eaa6..2446911f4262 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -237,7 +237,7 @@ SECTIONS
. += RESERVED_TTBR0_SIZE;
#endif
swapper_pg_dir = .;
- . += SWAPPER_DIR_SIZE;
+ . += PAGE_SIZE;
swapper_pg_end = .;
__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 088a591e4ea4..1d16104aa0b9 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -633,14 +633,6 @@ void __init paging_init(void)
map_mem(swapper_pg_dir);
cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
init_mm.pgd = swapper_pg_dir;
-
- /*
- * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
- * allocated with it.
- */
- memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE,
- __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir)
- - PAGE_SIZE);
}
void __init set_init_mm_pgd(pgd_t *pgd)
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [RESEND PATCH v4 5/6] arm64/mm: Populate swapper_pg_dir by
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
` (3 preceding siblings ...)
2018-07-18 10:17 ` [RESEND PATCH v4 4/6] arm64/mm: Make swapper_pg_dir smaller Jun Yao
@ 2018-07-18 10:17 ` Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 6/6] arm64/mm: Move Jun Yao
5 siblings, 0 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
To move swapper_pg_dir to .rodata section, we populate it by
fixmap.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
---
arch/arm64/include/asm/pgtable.h | 68 ++++++++++++++++++++++++++------
arch/arm64/mm/mmu.c | 2 +
2 files changed, 59 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 46ef21ebfe47..d5c3df99af7b 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -45,6 +45,13 @@
#include <linux/mm_types.h>
#include <linux/sched.h>
+extern pgd_t init_pg_dir[PTRS_PER_PGD];
+extern pgd_t init_pg_end[];
+extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
+extern pgd_t swapper_pg_end[];
+extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
+extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
+
extern void __pte_error(const char *file, int line, unsigned long val);
extern void __pmd_error(const char *file, int line, unsigned long val);
extern void __pud_error(const char *file, int line, unsigned long val);
@@ -428,8 +435,32 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
PUD_TYPE_TABLE)
#endif
+extern spinlock_t swapper_pgdir_lock;
+
+#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
+#define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
+
+static inline bool in_swapper_pgdir(void *addr)
+{
+ return ((unsigned long)addr & PAGE_MASK) ==
+ ((unsigned long)swapper_pg_dir & PAGE_MASK);
+}
+
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
{
+#ifdef __PAGETABLE_PMD_FOLDED
+ if (in_swapper_pgdir(pmdp)) {
+ pmd_t *fixmap_pmdp;
+
+ spin_lock(&swapper_pgdir_lock);
+ fixmap_pmdp = (pmd_t *)pgd_set_fixmap(__pa(pmdp));
+ WRITE_ONCE(*fixmap_pmdp, pmd);
+ dsb(ishst);
+ pgd_clear_fixmap();
+ spin_unlock(&swapper_pgdir_lock);
+ return;
+ }
+#endif
WRITE_ONCE(*pmdp, pmd);
dsb(ishst);
}
@@ -480,6 +511,19 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
static inline void set_pud(pud_t *pudp, pud_t pud)
{
+#ifdef __PAGETABLE_PUD_FOLDED
+ if (in_swapper_pgdir(pudp)) {
+ pud_t *fixmap_pudp;
+
+ spin_lock(&swapper_pgdir_lock);
+ fixmap_pudp = (pud_t *)pgd_set_fixmap(__pa(pudp));
+ WRITE_ONCE(*fixmap_pudp, pud);
+ dsb(ishst);
+ pgd_clear_fixmap();
+ spin_unlock(&swapper_pgdir_lock);
+ return;
+ }
+#endif
WRITE_ONCE(*pudp, pud);
dsb(ishst);
}
@@ -532,8 +576,19 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)
static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
{
- WRITE_ONCE(*pgdp, pgd);
- dsb(ishst);
+ if (in_swapper_pgdir(pgdp)) {
+ pgd_t *fixmap_pgdp;
+
+ spin_lock(&swapper_pgdir_lock);
+ fixmap_pgdp = pgd_set_fixmap(__pa(pgdp));
+ WRITE_ONCE(*fixmap_pgdp, pgd);
+ dsb(ishst);
+ pgd_clear_fixmap();
+ spin_unlock(&swapper_pgdir_lock);
+ } else {
+ WRITE_ONCE(*pgdp, pgd);
+ dsb(ishst);
+ }
}
static inline void pgd_clear(pgd_t *pgdp)
@@ -586,8 +641,6 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
/* to find an entry in a kernel page-table-directory */
#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
-#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
-#define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
@@ -712,13 +765,6 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
}
#endif
-extern pgd_t init_pg_dir[PTRS_PER_PGD];
-extern pgd_t init_pg_end[];
-extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
-extern pgd_t swapper_pg_end[];
-extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
-extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
-
/*
* Encode and decode a swap entry:
* bits 0-1: present (must be zero)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 1d16104aa0b9..7961ed8e6967 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -67,6 +67,8 @@ static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+DEFINE_SPINLOCK(swapper_pgdir_lock);
+
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
unsigned long size, pgprot_t vma_prot)
{
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [RESEND PATCH v4 6/6] arm64/mm: Move
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
` (4 preceding siblings ...)
2018-07-18 10:17 ` [RESEND PATCH v4 5/6] arm64/mm: Populate swapper_pg_dir by Jun Yao
@ 2018-07-18 10:17 ` Jun Yao
5 siblings, 0 replies; 7+ messages in thread
From: Jun Yao @ 2018-07-18 10:17 UTC (permalink / raw)
To: linux-arm-kernel; +Cc: catalin.marinas, will.deacon, james.morse, linux-kernel
Move {idmap_pg_dir, tramp_pg_dir, reserved_ttbr0, swapper_pg_dir}
to .rodata section.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
---
arch/arm64/kernel/vmlinux.lds.S | 39 ++++++++++++++++++++-------------
1 file changed, 24 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 2446911f4262..142528a23b44 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -64,8 +64,13 @@ jiffies = jiffies_64;
*(.entry.tramp.text) \
. = ALIGN(PAGE_SIZE); \
__entry_tramp_text_end = .;
+
+#define TRAMP_PG_TABLE \
+ tramp_pg_dir = .; \
+ . += PAGE_SIZE;
#else
#define TRAMP_TEXT
+#define TRAMP_PG_TABLE
#endif
#define INIT_PG_TABLES \
@@ -74,6 +79,24 @@ jiffies = jiffies_64;
. += SWAPPER_DIR_SIZE; \
init_pg_end = .;
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+#define RESERVED_PG_TABLE \
+ reserved_ttbr0 = .; \
+ . += RESERVED_TTBR0_SIZE;
+#else
+#define RESERVED_PG_TABLE
+#endif
+
+#define KERNEL_PG_TABLES \
+ . = ALIGN(PAGE_SIZE); \
+ idmap_pg_dir = .; \
+ . += IDMAP_DIR_SIZE; \
+ TRAMP_PG_TABLE \
+ RESERVED_PG_TABLE \
+ swapper_pg_dir = .; \
+ . += PAGE_SIZE; \
+ swapper_pg_end = .;
+
/*
* The size of the PE/COFF section that covers the kernel image, which
* runs from stext to _edata, must be a round multiple of the PE/COFF
@@ -143,6 +166,7 @@ SECTIONS
RO_DATA(PAGE_SIZE) /* everything from this point to */
EXCEPTION_TABLE(8) /* __init_begin will be marked RO NX */
NOTES
+ KERNEL_PG_TABLES
. = ALIGN(SEGMENT_ALIGN);
__init_begin = .;
@@ -224,21 +248,6 @@ SECTIONS
BSS_SECTION(0, 0, 0)
. = ALIGN(PAGE_SIZE);
- idmap_pg_dir = .;
- . += IDMAP_DIR_SIZE;
-
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
- tramp_pg_dir = .;
- . += PAGE_SIZE;
-#endif
-
-#ifdef CONFIG_ARM64_SW_TTBR0_PAN
- reserved_ttbr0 = .;
- . += RESERVED_TTBR0_SIZE;
-#endif
- swapper_pg_dir = .;
- . += PAGE_SIZE;
- swapper_pg_end = .;
__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
_end = .;
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
end of thread, other threads:[~2018-07-18 10:18 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-07-18 10:17 [RESEND PATCH v4 0/6] arm64/mm: Move Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 1/6] arm64/mm: Introduce init_pg_dir Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 2/6] arm64/mm: Make __enable_mmu() take the Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 3/6] arm64/mm: Create initial page tables in Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 4/6] arm64/mm: Make swapper_pg_dir smaller Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 5/6] arm64/mm: Populate swapper_pg_dir by Jun Yao
2018-07-18 10:17 ` [RESEND PATCH v4 6/6] arm64/mm: Move Jun Yao
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).