From: Yu-cheng Yu <yu-cheng.yu@intel.com> To: x86@kernel.org, "H. Peter Anvin" <hpa@zytor.com>, Thomas Gleixner <tglx@linutronix.de>, Ingo Molnar <mingo@redhat.com>, linux-kernel@vger.kernel.org, linux-doc@vger.kernel.org, linux-mm@kvack.org, linux-arch@vger.kernel.org, linux-api@vger.kernel.org, Arnd Bergmann <arnd@arndb.de>, Andy Lutomirski <luto@amacapital.net>, Balbir Singh <bsingharora@gmail.com>, Cyrill Gorcunov <gorcunov@gmail.com>, Dave Hansen <dave.hansen@linux.intel.com>, Florian Weimer <fweimer@redhat.com>, "H.J. Lu" <hjl.tools@gmail.com>, Jann Horn <jannh@google.com>, Jonathan Corbet <corbet@lwn.net>, Kees Cook <keescook@chromiun.org>, Mike Kravetz <mike.kravetz@oracle.com>, Nadav Amit <nadav.amit@gmail.com>, Oleg Nesterov <oleg@redhat.com>, Pavel Machek <pavel@ucw.cz>, Peter Zijlstra <peterz@infradead.org>, "Ravi V. Shankar" <ravi.v.shankar@intel.com>, Vedvyas Shanbhogue <vedvyas.shanbhogue@intel.com> Cc: Yu-cheng Yu <yu-cheng.yu@intel.com> Subject: [RFC PATCH v3 10/24] x86/mm: Introduce _PAGE_DIRTY_SW Date: Thu, 30 Aug 2018 07:38:50 -0700 [thread overview] Message-ID: <20180830143904.3168-11-yu-cheng.yu@intel.com> (raw) In-Reply-To: <20180830143904.3168-1-yu-cheng.yu@intel.com> A RO and dirty PTE exists in the following cases: (a) A page is modified and then shared with a fork()'ed child; (b) A R/O page that has been COW'ed; (c) A SHSTK page. The processor does not read the dirty bit for (a) and (b), but checks the dirty bit for (c). To prevent the use of non-SHSTK memory as SHSTK, we introduce a spare bit of the 64-bit PTE as _PAGE_BIT_DIRTY_SW and use that for (a) and (b). This results to the following possible PTE settings: Modified PTE: (R/W + DIRTY_HW) Modified and shared PTE: (R/O + DIRTY_SW) R/O PTE COW'ed: (R/O + DIRTY_SW) SHSTK PTE: (R/O + DIRTY_HW) SHSTK PTE COW'ed: (R/O + DIRTY_HW) SHSTK PTE shared: (R/O + DIRTY_SW) Note that _PAGE_BIT_DRITY_SW is only used in R/O PTEs but not R/W PTEs. When this patch is applied, there are six free bits left in the 64-bit PTE. There is no more free bit in the 32-bit PTE (except for PAE) and shadow stack is not implemented for the 32-bit kernel. Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com> --- arch/x86/include/asm/pgtable.h | 129 ++++++++++++++++++++++----- arch/x86/include/asm/pgtable_types.h | 14 ++- include/asm-generic/pgtable.h | 21 +++++ 3 files changed, 142 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index aab42464f6a1..4d50de77ea96 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -119,9 +119,9 @@ extern pmdval_t early_pmd_flags; * The following only work if pte_present() is true. * Undefined behaviour if not.. */ -static inline int pte_dirty(pte_t pte) +static inline bool pte_dirty(pte_t pte) { - return pte_flags(pte) & _PAGE_DIRTY; + return pte_flags(pte) & _PAGE_DIRTY_BITS; } @@ -143,9 +143,9 @@ static inline int pte_young(pte_t pte) return pte_flags(pte) & _PAGE_ACCESSED; } -static inline int pmd_dirty(pmd_t pmd) +static inline bool pmd_dirty(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_DIRTY; + return pmd_flags(pmd) & _PAGE_DIRTY_BITS; } static inline int pmd_young(pmd_t pmd) @@ -153,9 +153,9 @@ static inline int pmd_young(pmd_t pmd) return pmd_flags(pmd) & _PAGE_ACCESSED; } -static inline int pud_dirty(pud_t pud) +static inline bool pud_dirty(pud_t pud) { - return pud_flags(pud) & _PAGE_DIRTY; + return pud_flags(pud) & _PAGE_DIRTY_BITS; } static inline int pud_young(pud_t pud) @@ -294,9 +294,23 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) return native_make_pte(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + if (pte_flags(pte) & from) + pte = pte_set_flags(pte_clear_flags(pte, from), to); + return pte; +} +#else +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + return pte; +} +#endif + static inline pte_t pte_mkclean(pte_t pte) { - return pte_clear_flags(pte, _PAGE_DIRTY); + return pte_clear_flags(pte, _PAGE_DIRTY_BITS); } static inline pte_t pte_mkold(pte_t pte) @@ -306,6 +320,7 @@ static inline pte_t pte_mkold(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pte_clear_flags(pte, _PAGE_RW); } @@ -316,9 +331,24 @@ static inline pte_t pte_mkexec(pte_t pte) static inline pte_t pte_mkdirty(pte_t pte) { + pteval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHSTK_USER) || + pte_write(pte)) ? _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pte_set_flags(pte, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pte_t pte_mkdirty_shstk(pte_t pte) +{ + pte = pte_clear_flags(pte, _PAGE_DIRTY_SW); return pte_set_flags(pte, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pte_dirty_hw(pte_t pte) +{ + return pte_flags(pte) & _PAGE_DIRTY_HW; +} +#endif + static inline pte_t pte_mkyoung(pte_t pte) { return pte_set_flags(pte, _PAGE_ACCESSED); @@ -326,6 +356,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkwrite(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pte_set_flags(pte, _PAGE_RW); } @@ -373,6 +404,20 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) return native_make_pmd(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + if (pmd_flags(pmd) & from) + pmd = pmd_set_flags(pmd_clear_flags(pmd, from), to); + return pmd; +} +#else +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + return pmd; +} +#endif + static inline pmd_t pmd_mkold(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_ACCESSED); @@ -380,19 +425,36 @@ static inline pmd_t pmd_mkold(pmd_t pmd) static inline pmd_t pmd_mkclean(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_DIRTY); + return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS); } static inline pmd_t pmd_wrprotect(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pmd_clear_flags(pmd, _PAGE_RW); } static inline pmd_t pmd_mkdirty(pmd_t pmd) { + pmdval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHSTK_USER) || + (pmd_flags(pmd) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pmd_set_flags(pmd, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pmd_t pmd_mkdirty_shstk(pmd_t pmd) +{ + pmd = pmd_clear_flags(pmd, _PAGE_DIRTY_SW); return pmd_set_flags(pmd, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pmd_dirty_hw(pmd_t pmd) +{ + return pmd_flags(pmd) & _PAGE_DIRTY_HW; +} +#endif + static inline pmd_t pmd_mkdevmap(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_DEVMAP); @@ -410,6 +472,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) static inline pmd_t pmd_mkwrite(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pmd_set_flags(pmd, _PAGE_RW); } @@ -427,6 +490,20 @@ static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) return native_make_pud(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + if (pud_flags(pud) & from) + pud = pud_set_flags(pud_clear_flags(pud, from), to); + return pud; +} +#else +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + return pud; +} +#endif + static inline pud_t pud_mkold(pud_t pud) { return pud_clear_flags(pud, _PAGE_ACCESSED); @@ -434,17 +511,22 @@ static inline pud_t pud_mkold(pud_t pud) static inline pud_t pud_mkclean(pud_t pud) { - return pud_clear_flags(pud, _PAGE_DIRTY); + return pud_clear_flags(pud, _PAGE_DIRTY_BITS); } static inline pud_t pud_wrprotect(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pud_clear_flags(pud, _PAGE_RW); } static inline pud_t pud_mkdirty(pud_t pud) { - return pud_set_flags(pud, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); + pudval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHSTK_USER) || + (pud_flags(pud) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + + return pud_set_flags(pud, dirty | _PAGE_SOFT_DIRTY); } static inline pud_t pud_mkdevmap(pud_t pud) @@ -464,6 +546,7 @@ static inline pud_t pud_mkyoung(pud_t pud) static inline pud_t pud_mkwrite(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pud_set_flags(pud, _PAGE_RW); } @@ -595,19 +678,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) val &= _PAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK; val = flip_protnone_guard(oldval, val, PTE_PFN_MASK); + if ((pte_write(pte) && !(pgprot_val(newprot) & _PAGE_RW))) + return pte_move_flags(__pte(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); return __pte(val); } -static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) -{ - pmdval_t val = pmd_val(pmd), oldval = val; - - val &= _HPAGE_CHG_MASK; - val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; - val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); - return __pmd(val); -} - /* mprotect needs to preserve PAT bits when updating vm_page_prot */ #define pgprot_modify pgprot_modify static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) @@ -1159,6 +1235,19 @@ static inline int pmd_write(pmd_t pmd) return pmd_flags(pmd) & _PAGE_RW; } +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmdval_t val = pmd_val(pmd), oldval = val; + + val &= _HPAGE_CHG_MASK; + val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; + val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); + if ((pmd_write(pmd) && !(pgprot_val(newprot) & _PAGE_RW))) + return pmd_move_flags(__pmd(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); + return __pmd(val); +} + #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 0657a22d5216..f47bbc1f9c45 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -23,6 +23,7 @@ #define _PAGE_BIT_SOFTW2 10 /* " */ #define _PAGE_BIT_SOFTW3 11 /* " */ #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ +#define _PAGE_BIT_SOFTW5 57 /* available for programmer */ #define _PAGE_BIT_SOFTW4 58 /* available for programmer */ #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ @@ -34,6 +35,7 @@ #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 +#define _PAGE_BIT_DIRTY_SW _PAGE_BIT_SOFTW5 /* was written to */ /* If _PAGE_BIT_PRESENT is clear, we use these: */ /* - if the user mapped it with PROT_NONE; pte_present gives true */ @@ -109,6 +111,14 @@ #define _PAGE_DEVMAP (_AT(pteval_t, 0)) #endif +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +#define _PAGE_DIRTY_SW (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY_SW) +#else +#define _PAGE_DIRTY_SW (_AT(pteval_t, 0)) +#endif + +#define _PAGE_DIRTY_BITS (_PAGE_DIRTY_HW | _PAGE_DIRTY_SW) + #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) #define _PAGE_TABLE_NOENC (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |\ @@ -122,9 +132,9 @@ * instance, and is *not* included in this mask since * pte_modify() does modify it. */ -#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ +#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY_HW | \ - _PAGE_SOFT_DIRTY) + _PAGE_DIRTY_SW | _PAGE_SOFT_DIRTY) #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) /* diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 88ebc6102c7c..aa5271717126 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -1127,4 +1127,25 @@ static inline bool arch_has_pfn_modify_check(void) #endif #endif +#ifndef CONFIG_ARCH_HAS_SHSTK +static inline pte_t pte_mkdirty_shstk(pte_t pte) +{ + return pte; +} +static inline bool pte_dirty_hw(pte_t pte) +{ + return false; +} + +static inline pmd_t pmd_mkdirty_shstk(pmd_t pmd) +{ + return pmd; +} + +static inline bool pmd_dirty_hw(pmd_t pmd) +{ + return false; +} +#endif + #endif /* _ASM_GENERIC_PGTABLE_H */ -- 2.17.1
WARNING: multiple messages have this Message-ID (diff)
From: Yu-cheng Yu <yu-cheng.yu@intel.com> To: x86@kernel.org, "H. Peter Anvin" <hpa@zytor.com>, Thomas Gleixner <tglx@linutronix.de>, Ingo Molnar <mingo@redhat.com>, linux-kernel@vger.kernel.org, linux-doc@vger.kernel.org, linux-mm@kvack.org, linux-arch@vger.kernel.org, linux-api@vger.kernel.org, Arnd Bergmann <arnd@arndb.de>, Andy Lutomirski <luto@amacapital.net>, Balbir Singh <bsingharora@gmail.com>, Cyrill Gorcunov <gorcunov@gmail.com>, Dave Hansen <dave.hansen@linux.intel.com>, Florian Weimer <fweimer@redhat.com>, "H.J. Lu" <hjl.tools@gmail.com>, Jann Horn <jannh@google.com>, Jonathan Corbet <corbet@lwn.net>, Kees Cook <keescook@chromiun.org>, Mike Kravetz <mike.kravetz@oracle.com>, Nadav Amit <nadav.amit@gmail.com>, Oleg Nesterov <oleg@redhat.com>, Pavel Machek <pavel@ucw.cz>Peter Cc: Yu-cheng Yu <yu-cheng.yu@intel.com> Subject: [RFC PATCH v3 10/24] x86/mm: Introduce _PAGE_DIRTY_SW Date: Thu, 30 Aug 2018 07:38:50 -0700 [thread overview] Message-ID: <20180830143904.3168-11-yu-cheng.yu@intel.com> (raw) In-Reply-To: <20180830143904.3168-1-yu-cheng.yu@intel.com> A RO and dirty PTE exists in the following cases: (a) A page is modified and then shared with a fork()'ed child; (b) A R/O page that has been COW'ed; (c) A SHSTK page. The processor does not read the dirty bit for (a) and (b), but checks the dirty bit for (c). To prevent the use of non-SHSTK memory as SHSTK, we introduce a spare bit of the 64-bit PTE as _PAGE_BIT_DIRTY_SW and use that for (a) and (b). This results to the following possible PTE settings: Modified PTE: (R/W + DIRTY_HW) Modified and shared PTE: (R/O + DIRTY_SW) R/O PTE COW'ed: (R/O + DIRTY_SW) SHSTK PTE: (R/O + DIRTY_HW) SHSTK PTE COW'ed: (R/O + DIRTY_HW) SHSTK PTE shared: (R/O + DIRTY_SW) Note that _PAGE_BIT_DRITY_SW is only used in R/O PTEs but not R/W PTEs. When this patch is applied, there are six free bits left in the 64-bit PTE. There is no more free bit in the 32-bit PTE (except for PAE) and shadow stack is not implemented for the 32-bit kernel. Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com> --- arch/x86/include/asm/pgtable.h | 129 ++++++++++++++++++++++----- arch/x86/include/asm/pgtable_types.h | 14 ++- include/asm-generic/pgtable.h | 21 +++++ 3 files changed, 142 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index aab42464f6a1..4d50de77ea96 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -119,9 +119,9 @@ extern pmdval_t early_pmd_flags; * The following only work if pte_present() is true. * Undefined behaviour if not.. */ -static inline int pte_dirty(pte_t pte) +static inline bool pte_dirty(pte_t pte) { - return pte_flags(pte) & _PAGE_DIRTY; + return pte_flags(pte) & _PAGE_DIRTY_BITS; } @@ -143,9 +143,9 @@ static inline int pte_young(pte_t pte) return pte_flags(pte) & _PAGE_ACCESSED; } -static inline int pmd_dirty(pmd_t pmd) +static inline bool pmd_dirty(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_DIRTY; + return pmd_flags(pmd) & _PAGE_DIRTY_BITS; } static inline int pmd_young(pmd_t pmd) @@ -153,9 +153,9 @@ static inline int pmd_young(pmd_t pmd) return pmd_flags(pmd) & _PAGE_ACCESSED; } -static inline int pud_dirty(pud_t pud) +static inline bool pud_dirty(pud_t pud) { - return pud_flags(pud) & _PAGE_DIRTY; + return pud_flags(pud) & _PAGE_DIRTY_BITS; } static inline int pud_young(pud_t pud) @@ -294,9 +294,23 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) return native_make_pte(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + if (pte_flags(pte) & from) + pte = pte_set_flags(pte_clear_flags(pte, from), to); + return pte; +} +#else +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + return pte; +} +#endif + static inline pte_t pte_mkclean(pte_t pte) { - return pte_clear_flags(pte, _PAGE_DIRTY); + return pte_clear_flags(pte, _PAGE_DIRTY_BITS); } static inline pte_t pte_mkold(pte_t pte) @@ -306,6 +320,7 @@ static inline pte_t pte_mkold(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pte_clear_flags(pte, _PAGE_RW); } @@ -316,9 +331,24 @@ static inline pte_t pte_mkexec(pte_t pte) static inline pte_t pte_mkdirty(pte_t pte) { + pteval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHSTK_USER) || + pte_write(pte)) ? _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pte_set_flags(pte, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pte_t pte_mkdirty_shstk(pte_t pte) +{ + pte = pte_clear_flags(pte, _PAGE_DIRTY_SW); return pte_set_flags(pte, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pte_dirty_hw(pte_t pte) +{ + return pte_flags(pte) & _PAGE_DIRTY_HW; +} +#endif + static inline pte_t pte_mkyoung(pte_t pte) { return pte_set_flags(pte, _PAGE_ACCESSED); @@ -326,6 +356,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkwrite(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pte_set_flags(pte, _PAGE_RW); } @@ -373,6 +404,20 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) return native_make_pmd(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + if (pmd_flags(pmd) & from) + pmd = pmd_set_flags(pmd_clear_flags(pmd, from), to); + return pmd; +} +#else +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + return pmd; +} +#endif + static inline pmd_t pmd_mkold(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_ACCESSED); @@ -380,19 +425,36 @@ static inline pmd_t pmd_mkold(pmd_t pmd) static inline pmd_t pmd_mkclean(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_DIRTY); + return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS); } static inline pmd_t pmd_wrprotect(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pmd_clear_flags(pmd, _PAGE_RW); } static inline pmd_t pmd_mkdirty(pmd_t pmd) { + pmdval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHSTK_USER) || + (pmd_flags(pmd) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pmd_set_flags(pmd, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pmd_t pmd_mkdirty_shstk(pmd_t pmd) +{ + pmd = pmd_clear_flags(pmd, _PAGE_DIRTY_SW); return pmd_set_flags(pmd, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pmd_dirty_hw(pmd_t pmd) +{ + return pmd_flags(pmd) & _PAGE_DIRTY_HW; +} +#endif + static inline pmd_t pmd_mkdevmap(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_DEVMAP); @@ -410,6 +472,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) static inline pmd_t pmd_mkwrite(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pmd_set_flags(pmd, _PAGE_RW); } @@ -427,6 +490,20 @@ static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) return native_make_pud(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + if (pud_flags(pud) & from) + pud = pud_set_flags(pud_clear_flags(pud, from), to); + return pud; +} +#else +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + return pud; +} +#endif + static inline pud_t pud_mkold(pud_t pud) { return pud_clear_flags(pud, _PAGE_ACCESSED); @@ -434,17 +511,22 @@ static inline pud_t pud_mkold(pud_t pud) static inline pud_t pud_mkclean(pud_t pud) { - return pud_clear_flags(pud, _PAGE_DIRTY); + return pud_clear_flags(pud, _PAGE_DIRTY_BITS); } static inline pud_t pud_wrprotect(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pud_clear_flags(pud, _PAGE_RW); } static inline pud_t pud_mkdirty(pud_t pud) { - return pud_set_flags(pud, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); + pudval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHSTK_USER) || + (pud_flags(pud) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + + return pud_set_flags(pud, dirty | _PAGE_SOFT_DIRTY); } static inline pud_t pud_mkdevmap(pud_t pud) @@ -464,6 +546,7 @@ static inline pud_t pud_mkyoung(pud_t pud) static inline pud_t pud_mkwrite(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pud_set_flags(pud, _PAGE_RW); } @@ -595,19 +678,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) val &= _PAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK; val = flip_protnone_guard(oldval, val, PTE_PFN_MASK); + if ((pte_write(pte) && !(pgprot_val(newprot) & _PAGE_RW))) + return pte_move_flags(__pte(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); return __pte(val); } -static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) -{ - pmdval_t val = pmd_val(pmd), oldval = val; - - val &= _HPAGE_CHG_MASK; - val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; - val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); - return __pmd(val); -} - /* mprotect needs to preserve PAT bits when updating vm_page_prot */ #define pgprot_modify pgprot_modify static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) @@ -1159,6 +1235,19 @@ static inline int pmd_write(pmd_t pmd) return pmd_flags(pmd) & _PAGE_RW; } +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmdval_t val = pmd_val(pmd), oldval = val; + + val &= _HPAGE_CHG_MASK; + val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; + val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); + if ((pmd_write(pmd) && !(pgprot_val(newprot) & _PAGE_RW))) + return pmd_move_flags(__pmd(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); + return __pmd(val); +} + #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 0657a22d5216..f47bbc1f9c45 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -23,6 +23,7 @@ #define _PAGE_BIT_SOFTW2 10 /* " */ #define _PAGE_BIT_SOFTW3 11 /* " */ #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ +#define _PAGE_BIT_SOFTW5 57 /* available for programmer */ #define _PAGE_BIT_SOFTW4 58 /* available for programmer */ #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ @@ -34,6 +35,7 @@ #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 +#define _PAGE_BIT_DIRTY_SW _PAGE_BIT_SOFTW5 /* was written to */ /* If _PAGE_BIT_PRESENT is clear, we use these: */ /* - if the user mapped it with PROT_NONE; pte_present gives true */ @@ -109,6 +111,14 @@ #define _PAGE_DEVMAP (_AT(pteval_t, 0)) #endif +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +#define _PAGE_DIRTY_SW (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY_SW) +#else +#define _PAGE_DIRTY_SW (_AT(pteval_t, 0)) +#endif + +#define _PAGE_DIRTY_BITS (_PAGE_DIRTY_HW | _PAGE_DIRTY_SW) + #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) #define _PAGE_TABLE_NOENC (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |\ @@ -122,9 +132,9 @@ * instance, and is *not* included in this mask since * pte_modify() does modify it. */ -#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ +#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY_HW | \ - _PAGE_SOFT_DIRTY) + _PAGE_DIRTY_SW | _PAGE_SOFT_DIRTY) #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) /* diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 88ebc6102c7c..aa5271717126 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -1127,4 +1127,25 @@ static inline bool arch_has_pfn_modify_check(void) #endif #endif +#ifndef CONFIG_ARCH_HAS_SHSTK +static inline pte_t pte_mkdirty_shstk(pte_t pte) +{ + return pte; +} +static inline bool pte_dirty_hw(pte_t pte) +{ + return false; +} + +static inline pmd_t pmd_mkdirty_shstk(pmd_t pmd) +{ + return pmd; +} + +static inline bool pmd_dirty_hw(pmd_t pmd) +{ + return false; +} +#endif + #endif /* _ASM_GENERIC_PGTABLE_H */ -- 2.17.1
next prev parent reply other threads:[~2018-08-30 14:43 UTC|newest] Thread overview: 167+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-08-30 14:38 [RFC PATCH v3 00/24] Control Flow Enforcement: Shadow Stack Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 01/24] x86/cpufeatures: Add CPUIDs for Control-flow Enforcement Technology (CET) Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 02/24] x86/fpu/xstate: Change some names to separate XSAVES system and user states Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 03/24] x86/fpu/xstate: Enable XSAVES system states Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 04/24] x86/fpu/xstate: Add XSAVES system states for shadow stack Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 05/24] Documentation/x86: Add CET description Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 20:39 ` Pavel Machek 2018-08-30 20:39 ` Pavel Machek 2018-08-30 22:49 ` Yu-cheng Yu 2018-08-30 22:49 ` Yu-cheng Yu 2018-08-30 22:49 ` Yu-cheng Yu 2018-09-14 21:17 ` Yu-cheng Yu 2018-09-14 21:17 ` Yu-cheng Yu 2018-09-14 21:17 ` Yu-cheng Yu 2018-09-03 2:56 ` Randy Dunlap 2018-09-03 2:56 ` Randy Dunlap 2018-08-30 14:38 ` [RFC PATCH v3 06/24] x86/cet: Control protection exception handler Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-31 15:01 ` Jann Horn 2018-08-31 15:01 ` Jann Horn 2018-08-31 16:20 ` Yu-cheng Yu 2018-08-31 16:20 ` Yu-cheng Yu 2018-08-31 16:20 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 07/24] x86/cet/shstk: Add Kconfig option for user-mode shadow stack Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 08/24] mm: Introduce VM_SHSTK for shadow stack memory Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 09/24] x86/mm: Change _PAGE_DIRTY to _PAGE_DIRTY_HW Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu [this message] 2018-08-30 14:38 ` [RFC PATCH v3 10/24] x86/mm: Introduce _PAGE_DIRTY_SW Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 11/24] drm/i915/gvt: Update _PAGE_DIRTY to _PAGE_DIRTY_BITS Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 12/24] x86/mm: Modify ptep_set_wrprotect and pmdp_set_wrprotect for _PAGE_DIRTY_SW Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 15:49 ` Jann Horn 2018-08-30 15:49 ` Jann Horn 2018-08-30 16:02 ` Yu-cheng Yu 2018-08-30 16:02 ` Yu-cheng Yu 2018-08-30 16:02 ` Yu-cheng Yu 2018-08-30 16:08 ` Dave Hansen 2018-08-30 16:08 ` Dave Hansen 2018-08-30 16:23 ` Jann Horn 2018-08-30 16:23 ` Jann Horn 2018-08-30 17:19 ` Dave Hansen 2018-08-30 17:19 ` Dave Hansen 2018-08-30 17:26 ` Yu-cheng Yu 2018-08-30 17:26 ` Yu-cheng Yu 2018-08-30 17:26 ` Yu-cheng Yu 2018-08-30 17:33 ` Dave Hansen 2018-08-30 17:33 ` Dave Hansen 2018-08-30 17:54 ` Yu-cheng Yu 2018-08-30 17:54 ` Yu-cheng Yu 2018-08-30 17:54 ` Yu-cheng Yu 2018-08-30 17:59 ` Jann Horn 2018-08-30 17:59 ` Jann Horn 2018-08-30 20:21 ` Yu-cheng Yu 2018-08-30 20:21 ` Yu-cheng Yu 2018-08-30 20:21 ` Yu-cheng Yu 2018-08-30 20:44 ` Jann Horn 2018-08-30 20:44 ` Jann Horn 2018-08-30 20:52 ` Yu-cheng Yu 2018-08-30 20:52 ` Yu-cheng Yu 2018-08-30 20:52 ` Yu-cheng Yu 2018-08-30 21:01 ` Jann Horn 2018-08-30 21:01 ` Jann Horn 2018-08-30 21:47 ` Jann Horn 2018-08-30 21:47 ` Jann Horn 2018-08-31 9:53 ` Peter Zijlstra 2018-08-31 9:53 ` Peter Zijlstra 2018-08-31 14:33 ` Yu-cheng Yu 2018-08-31 14:33 ` Yu-cheng Yu 2018-08-31 14:33 ` Yu-cheng Yu 2018-08-31 14:47 ` Dave Hansen 2018-08-31 14:47 ` Dave Hansen 2018-08-31 15:48 ` Yu-cheng Yu 2018-08-31 15:48 ` Yu-cheng Yu 2018-08-31 15:48 ` Yu-cheng Yu 2018-08-31 15:58 ` Dave Hansen 2018-08-31 15:58 ` Dave Hansen 2018-08-31 16:29 ` Peter Zijlstra 2018-08-31 16:29 ` Peter Zijlstra 2018-09-14 20:39 ` Yu-cheng Yu 2018-09-14 20:39 ` Yu-cheng Yu 2018-09-14 20:39 ` Yu-cheng Yu 2018-09-14 20:46 ` Dave Hansen 2018-09-14 20:46 ` Dave Hansen 2018-09-14 20:46 ` Dave Hansen 2018-09-14 21:08 ` Yu-cheng Yu 2018-09-14 21:08 ` Yu-cheng Yu 2018-09-14 21:08 ` Yu-cheng Yu 2018-09-14 21:33 ` Dave Hansen 2018-09-14 21:33 ` Dave Hansen 2018-09-14 21:33 ` Dave Hansen 2018-08-31 1:23 ` Andy Lutomirski 2018-08-31 1:23 ` Andy Lutomirski 2018-08-30 17:34 ` Andy Lutomirski 2018-08-30 17:34 ` Andy Lutomirski 2018-08-30 18:55 ` Dave Hansen 2018-08-30 18:55 ` Dave Hansen 2018-08-31 17:46 ` Andy Lutomirski 2018-08-31 17:46 ` Andy Lutomirski 2018-08-31 17:52 ` Dave Hansen 2018-08-31 17:52 ` Dave Hansen 2018-08-31 17:52 ` Dave Hansen 2018-08-30 19:59 ` Randy Dunlap 2018-08-30 19:59 ` Randy Dunlap 2018-08-30 20:23 ` Yu-cheng Yu 2018-08-30 20:23 ` Yu-cheng Yu 2018-08-30 20:23 ` Yu-cheng Yu 2018-08-31 16:29 ` Dave Hansen 2018-08-31 16:29 ` Dave Hansen 2018-08-30 14:38 ` [RFC PATCH v3 13/24] x86/mm: Shadow stack page fault error checking Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 14/24] mm: Handle shadow stack page fault Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 15/24] mm: Handle THP/HugeTLB " Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 16/24] mm: Update can_follow_write_pte/pmd for shadow stack Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 17/24] mm: Introduce do_mmap_locked() Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 18/24] x86/cet/shstk: User-mode shadow stack support Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 16:10 ` Jann Horn 2018-08-30 16:10 ` Jann Horn 2018-08-30 16:20 ` Yu-cheng Yu 2018-08-30 16:20 ` Yu-cheng Yu 2018-08-30 16:20 ` Yu-cheng Yu 2018-08-30 14:38 ` [RFC PATCH v3 19/24] x86/cet/shstk: Introduce WRUSS instruction Yu-cheng Yu 2018-08-30 14:38 ` Yu-cheng Yu 2018-08-30 15:39 ` Jann Horn 2018-08-30 15:39 ` Jann Horn 2018-08-30 15:55 ` Andy Lutomirski 2018-08-30 15:55 ` Andy Lutomirski 2018-08-30 16:22 ` Yu-cheng Yu 2018-08-30 16:22 ` Yu-cheng Yu 2018-08-30 16:22 ` Yu-cheng Yu 2018-08-31 21:49 ` Yu-cheng Yu 2018-08-31 21:49 ` Yu-cheng Yu 2018-08-31 21:49 ` Yu-cheng Yu 2018-08-31 22:16 ` Andy Lutomirski 2018-08-31 22:16 ` Andy Lutomirski 2018-09-14 20:46 ` Yu-cheng Yu 2018-09-14 20:46 ` Yu-cheng Yu 2018-09-14 20:46 ` Yu-cheng Yu 2018-08-30 14:39 ` [RFC PATCH v3 20/24] x86/cet/shstk: Signal handling for shadow stack Yu-cheng Yu 2018-08-30 14:39 ` Yu-cheng Yu 2018-08-30 14:39 ` [RFC PATCH v3 21/24] x86/cet/shstk: ELF header parsing of Shadow Stack Yu-cheng Yu 2018-08-30 14:39 ` Yu-cheng Yu 2018-08-30 14:39 ` [RFC PATCH v3 22/24] x86/cet/shstk: Handle thread shadow stack Yu-cheng Yu 2018-08-30 14:39 ` Yu-cheng Yu 2018-08-30 14:39 ` [RFC PATCH v3 23/24] x86/cet/shstk: Add arch_prctl functions for Shadow Stack Yu-cheng Yu 2018-08-30 14:39 ` Yu-cheng Yu 2018-08-30 14:39 ` [RFC PATCH v3 24/24] x86/cet/shstk: Add Shadow Stack instructions to opcode map Yu-cheng Yu 2018-08-30 14:39 ` Yu-cheng Yu 2018-09-02 8:13 ` [RFC PATCH v3 00/24] Control Flow Enforcement: Shadow Stack Balbir Singh 2018-09-02 8:13 ` Balbir Singh 2018-09-04 14:47 ` Yu-cheng Yu 2018-09-04 14:47 ` Yu-cheng Yu 2018-09-04 14:47 ` Yu-cheng Yu
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20180830143904.3168-11-yu-cheng.yu@intel.com \ --to=yu-cheng.yu@intel.com \ --cc=arnd@arndb.de \ --cc=bsingharora@gmail.com \ --cc=corbet@lwn.net \ --cc=dave.hansen@linux.intel.com \ --cc=fweimer@redhat.com \ --cc=gorcunov@gmail.com \ --cc=hjl.tools@gmail.com \ --cc=hpa@zytor.com \ --cc=jannh@google.com \ --cc=keescook@chromiun.org \ --cc=linux-api@vger.kernel.org \ --cc=linux-arch@vger.kernel.org \ --cc=linux-doc@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=luto@amacapital.net \ --cc=mike.kravetz@oracle.com \ --cc=mingo@redhat.com \ --cc=nadav.amit@gmail.com \ --cc=oleg@redhat.com \ --cc=pavel@ucw.cz \ --cc=peterz@infradead.org \ --cc=ravi.v.shankar@intel.com \ --cc=tglx@linutronix.de \ --cc=vedvyas.shanbhogue@intel.com \ --cc=x86@kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.