From: Michael Ellerman <mpe@ellerman.id.au>
To: <linuxppc-dev@ozlabs.org>
Cc: aneesh.kumar@linux.vnet.ibm.com, bsingharora@gmail.com,
Paul Mackerras <paulus@samba.org>
Subject: [PATCH v3 67/70] powerpc/mm/radix: Add radix THP callbacks
Date: Fri, 29 Apr 2016 23:26:30 +1000 [thread overview]
Message-ID: <1461936393-10131-67-git-send-email-mpe@ellerman.id.au> (raw)
In-Reply-To: <1461936393-10131-1-git-send-email-mpe@ellerman.id.au>
From: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
The deposited pgtable_t is a pte fragment hence we cannot use page->lru
for linking then together. We use the first two 64 bits for pte fragment
as list_head type to link all deposited fragments together. On withdraw
we properly zero then out.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/book3s/64/pgtable-64k.h | 2 +
arch/powerpc/include/asm/book3s/64/pgtable.h | 16 ++++
arch/powerpc/include/asm/book3s/64/radix.h | 22 +++++
arch/powerpc/mm/pgtable-book3s64.c | 2 +-
arch/powerpc/mm/pgtable-radix.c | 117 +++++++++++++++++++++++
5 files changed, 158 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
index 79331cf77613..793ecd5ea301 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
@@ -71,6 +71,8 @@ static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
+ if (radix_enabled())
+ return radix__pmd_mkhuge(pmd);
return hash__pmd_mkhuge(pmd);
}
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 5f290d39e563..1b09ba0c2d1d 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -827,6 +827,8 @@ extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
extern int hash__has_transparent_hugepage(void);
static inline int has_transparent_hugepage(void)
{
+ if (radix_enabled())
+ return radix__has_transparent_hugepage();
return hash__has_transparent_hugepage();
}
@@ -834,6 +836,8 @@ static inline unsigned long
pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp,
unsigned long clr, unsigned long set)
{
+ if (radix_enabled())
+ return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set);
return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set);
}
@@ -885,12 +889,16 @@ extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
+ if (radix_enabled())
+ return radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
return hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
}
static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
+ if (radix_enabled())
+ return radix__pmdp_collapse_flush(vma, address, pmdp);
return hash__pmdp_collapse_flush(vma, address, pmdp);
}
#define pmdp_collapse_flush pmdp_collapse_flush
@@ -899,6 +907,8 @@ static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
static inline void pgtable_trans_huge_deposit(struct mm_struct *mm,
pmd_t *pmdp, pgtable_t pgtable)
{
+ if (radix_enabled())
+ return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
}
@@ -906,6 +916,8 @@ static inline void pgtable_trans_huge_deposit(struct mm_struct *mm,
static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm,
pmd_t *pmdp)
{
+ if (radix_enabled())
+ return radix__pgtable_trans_huge_withdraw(mm, pmdp);
return hash__pgtable_trans_huge_withdraw(mm, pmdp);
}
@@ -917,6 +929,8 @@ extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
static inline void pmdp_huge_split_prepare(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
+ if (radix_enabled())
+ return radix__pmdp_huge_split_prepare(vma, address, pmdp);
return hash__pmdp_huge_split_prepare(vma, address, pmdp);
}
@@ -925,6 +939,8 @@ struct spinlock;
static inline int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
struct spinlock *old_pmd_ptl)
{
+ if (radix_enabled())
+ return false;
/*
* Archs like ppc64 use pgtable to store per pmd
* specific information. So when we switch the pmd,
diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h
index f4709024b466..937d4e247ac3 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -196,6 +196,28 @@ static inline int radix__pmd_trans_huge(pmd_t pmd)
return !!(pmd_val(pmd) & _PAGE_PTE);
}
+static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)
+{
+ return __pmd(pmd_val(pmd) | _PAGE_PTE);
+}
+static inline void radix__pmdp_huge_split_prepare(struct vm_area_struct *vma,
+ unsigned long address, pmd_t *pmdp)
+{
+ /* Nothing to do for radix. */
+ return;
+}
+
+extern unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp, unsigned long clr,
+ unsigned long set);
+extern pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma,
+ unsigned long address, pmd_t *pmdp);
+extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
+ pgtable_t pgtable);
+extern pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
+extern pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
+ unsigned long addr, pmd_t *pmdp);
+extern int radix__has_transparent_hugepage(void);
#endif
extern int __meminit radix__vmemmap_create_mapping(unsigned long start,
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index d566a250164d..eb4451144746 100644
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -69,7 +69,7 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
-
+ flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
/*
* This ensures that generic code that rely on IRQ disabling
* to prevent a parallel THP split work as expected.
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index a5a5253c3ea1..18b2c11604fa 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -19,6 +19,8 @@
#include <asm/mmu.h>
#include <asm/firmware.h>
+#include <trace/events/thp.h>
+
static int native_update_partition_table(u64 patb1)
{
partition_tb->patb1 = cpu_to_be64(patb1);
@@ -407,3 +409,118 @@ void radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
}
#endif
#endif
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+
+unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp, unsigned long clr,
+ unsigned long set)
+{
+ unsigned long old;
+
+#ifdef CONFIG_DEBUG_VM
+ WARN_ON(!radix__pmd_trans_huge(*pmdp));
+ assert_spin_locked(&mm->page_table_lock);
+#endif
+
+ old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
+ trace_hugepage_update(addr, old, clr, set);
+
+ return old;
+}
+
+pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmdp)
+
+{
+ pmd_t pmd;
+
+ VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+ VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
+ /*
+ * khugepaged calls this for normal pmd
+ */
+ pmd = *pmdp;
+ pmd_clear(pmdp);
+ /*FIXME!! Verify whether we need this kick below */
+ kick_all_cpus_sync();
+ flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+ return pmd;
+}
+
+/*
+ * For us pgtable_t is pte_t *. Inorder to save the deposisted
+ * page table, we consider the allocated page table as a list
+ * head. On withdraw we need to make sure we zero out the used
+ * list_head memory area.
+ */
+void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
+ pgtable_t pgtable)
+{
+ struct list_head *lh = (struct list_head *) pgtable;
+
+ assert_spin_locked(pmd_lockptr(mm, pmdp));
+
+ /* FIFO */
+ if (!pmd_huge_pte(mm, pmdp))
+ INIT_LIST_HEAD(lh);
+ else
+ list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
+ pmd_huge_pte(mm, pmdp) = pgtable;
+}
+
+pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
+{
+ pte_t *ptep;
+ pgtable_t pgtable;
+ struct list_head *lh;
+
+ assert_spin_locked(pmd_lockptr(mm, pmdp));
+
+ /* FIFO */
+ pgtable = pmd_huge_pte(mm, pmdp);
+ lh = (struct list_head *) pgtable;
+ if (list_empty(lh))
+ pmd_huge_pte(mm, pmdp) = NULL;
+ else {
+ pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
+ list_del(lh);
+ }
+ ptep = (pte_t *) pgtable;
+ *ptep = __pte(0);
+ ptep++;
+ *ptep = __pte(0);
+ return pgtable;
+}
+
+
+pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
+ unsigned long addr, pmd_t *pmdp)
+{
+ pmd_t old_pmd;
+ unsigned long old;
+
+ old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
+ old_pmd = __pmd(old);
+ /*
+ * Serialize against find_linux_pte_or_hugepte which does lock-less
+ * lookup in page tables with local interrupts disabled. For huge pages
+ * it casts pmd_t to pte_t. Since format of pte_t is different from
+ * pmd_t we want to prevent transit from pmd pointing to page table
+ * to pmd pointing to huge page (and back) while interrupts are disabled.
+ * We clear pmd to possibly replace it with page table pointer in
+ * different code paths. So make sure we wait for the parallel
+ * find_linux_pte_or_hugepage to finish.
+ */
+ kick_all_cpus_sync();
+ return old_pmd;
+}
+
+int radix__has_transparent_hugepage(void)
+{
+ /* For radix 2M at PMD level means thp */
+ if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT)
+ return 1;
+ return 0;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
--
2.5.0
next prev parent reply other threads:[~2016-04-29 13:27 UTC|newest]
Thread overview: 72+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-04-29 13:25 [PATCH v3 01/70] IB/qib: Use cache inhibitted and guarded mapping on powerpc Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 02/70] powerpc/mm: Always use STRICT_MM_TYPECHECKS Michael Ellerman
2016-05-01 13:02 ` [v3,02/70] " Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 03/70] powerpc/mm: Drop PTE_ATOMIC_UPDATES from pmd_hugepage_update() Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 04/70] powerpc/mm: Add pte_xchg() helper Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 05/70] powerpc/mm: Use big endian Linux page tables for book3s 64 Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 06/70] powerpc/mm: Use pte_raw() in pte_same()/pmd_same() Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 07/70] powerpc/mm: Use _PAGE_READ to indicate Read access Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 08/70] powerpc/mm/subpage: Clear RWX bit to indicate no access Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 09/70] powerpc/mm: Convert pte_user() to static inline Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 10/70] powerpc/mm: Use pte_user() instead of open coding Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 11/70] powerpc/mm: Replace _PAGE_USER with _PAGE_PRIVILEGED Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 12/70] powerpc/mm: Remove RPN_SHIFT and RPN_SIZE Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 13/70] powerpc/mm: Update _PAGE_KERNEL_RO Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 14/70] powerpc/mm: Use a helper for finding pte bits mapping I/O area Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 15/70] powerpc/mm: Drop WIMG in favour of new constants Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 16/70] powerpc/mm: Use generic version of pmdp_clear_flush_young() Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 17/70] powerpc/mm: Use generic version of ptep_clear_flush_young() Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 18/70] powerpc/mm: Move radix/hash common data structures to book3s64 headers Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 19/70] powerpc/mm/radix: Add partition table format & callback Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 20/70] powerpc/mm/hash: Add support for Power9 Hash Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 21/70] powerpc/mm: Move hash and no hash code to separate files Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 22/70] powerpc/mm/book3s: Rename hash specific PTE bits to carry H_ prefix Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 23/70] powerpc/mm: Handle _PTE_NONE_MASK Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 24/70] powerpc/mm: Move common pte bits and accessors to book3s/64/pgtable.h Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 25/70] powerpc/mm: Move pte accessors that operate on common pte bits to pgtable.h Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 26/70] powerpc/mm: Make page table size a variable Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 27/70] powerpc/mm: Move page table index and and vaddr to pgtable.h Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 28/70] powerpc/mm: Move pte related functions together Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 29/70] powerpc/mm/radix: Add radix pte #defines Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 30/70] powerpc/mm/radix: Add dummy radix_enabled() Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 31/70] powerpc/mm: Add radix callbacks to pte accessors Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 32/70] powerpc/mm: Move hugetlb and THP related pmd accessors to pgtable.h Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 33/70] powerpc/mm/radix: Add radix callback for pmd accessors Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 34/70] powerpc/mm: Abstract early MMU init in preparation for radix Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 35/70] powerpc/mm/radix: Add radix callbacks for early init routines Michael Ellerman
2016-04-29 13:25 ` [PATCH v3 36/70] powerpc/mm: Abstraction for vmemmap and map_kernel_page() Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 37/70] powerpc/mm/radix: Add radix callbacks for vmemmap and map_kernel page() Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 38/70] powerpc/mm: Abstraction for switch_mmu_context() Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 39/70] powerpc/mm/radix: Add mmu context handling callback for radix Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 40/70] powerpc/mm: Rename mmu_context_hash64.c to mmu_context_book3s64.c Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 41/70] powerpc/mm: Hash abstraction for tlbflush routines Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 42/70] powerpc/mm/radix: Add " Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 43/70] powerpc/mm/radix: Add MMU_FTR_RADIX Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 44/70] powerpc/mm/radix: Use STD_MMU_64 to properly isolate hash related code Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 45/70] powerpc/mm/radix: Isolate hash table function from pseries guest code Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 46/70] powerpc/mm/radix: Add checks in slice code to catch radix usage Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 47/70] powerpc/mm/radix: Limit paca allocation in radix Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 48/70] powerpc/mm/radix: Pick the address layout for radix config Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 49/70] powerpc/mm/radix: Update PTCR on secondary CPUs Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 50/70] powerpc/mm: Make a copy of pgalloc.h for 32 and 64 book3s Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 51/70] powerpc/mm: Copy pgalloc (part 2) Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 52/70] powerpc/mm: Revert changes made to nohash pgalloc-64.h Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 53/70] powerpc/mm: Simplify the code dropping 4-level table #ifdef Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 54/70] powerpc/mm: Rename function to indicate we are allocating fragments Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 55/70] powerpc/mm: Make 4K and 64K use pte_t for pgtable_t Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 56/70] powerpc/mm: Add radix pgalloc details Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 57/70] powerpc/mm: Update pte filter for radix Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 58/70] powerpc/mm: vmalloc abstraction in preparation " Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 59/70] powerpc/radix: Update MMU cache Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 60/70] powerpc/mm: pte_frag abstraction Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 61/70] powerpc/mm: Fix vma_mmu_pagesize() for radix Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 62/70] powerpc/mm: Add radix support for hugetlb Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 63/70] powerpc/mm/radix: Make sure swapper pgdir is properly aligned Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 64/70] powerpc/mm/radix: Add hugetlb support 4K page size Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 65/70] powerpc/mm: THP is only available on hash64 as of now Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 66/70] powerpc/mm/thp: Abstraction for THP functions Michael Ellerman
2016-04-29 13:26 ` Michael Ellerman [this message]
2016-04-29 13:26 ` [PATCH v3 68/70] powerpc/mm/radix: Add THP support for 4K linux page size Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 69/70] powerpc/mm/radix: Use firmware feature to enable Radix MMU Michael Ellerman
2016-04-29 13:26 ` [PATCH v3 70/70] powerpc/mm/radix: Document software bits for radix Michael Ellerman
2016-05-01 13:02 ` [v3, 01/70] IB/qib: Use cache inhibitted and guarded mapping on powerpc Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1461936393-10131-67-git-send-email-mpe@ellerman.id.au \
--to=mpe@ellerman.id.au \
--cc=aneesh.kumar@linux.vnet.ibm.com \
--cc=bsingharora@gmail.com \
--cc=linuxppc-dev@ozlabs.org \
--cc=paulus@samba.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).