From: Yu-cheng Yu <yu-cheng.yu@intel.com> To: x86@kernel.org, "H. Peter Anvin" <hpa@zytor.com>, Thomas Gleixner <tglx@linutronix.de>, Ingo Molnar <mingo@redhat.com>, linux-kernel@vger.kernel.org, linux-doc@vger.kernel.org, linux-mm@kvack.org, linux-arch@vger.kernel.org, linux-api@vger.kernel.org, Arnd Bergmann <arnd@arndb.de>, Andy Lutomirski <luto@amacapital.net>, Balbir Singh <bsingharora@gmail.com>, Cyrill Gorcunov <gorcunov@gmail.com>, Dave Hansen <dave.hansen@linux.intel.com>, Florian Weimer <fweimer@redhat.com>, "H.J. Lu" <hjl.tools@gmail.com>, Jann Horn <jannh@google.com>, Jonathan Corbet <corbet@lwn.net>, Kees Cook <keescook@chromium.org>, Mike Kravetz <mike.kravetz@oracle.com>, Nadav Amit <nadav.amit@gmail.com>, Oleg Nesterov <oleg@redhat.com>, Pavel Machek <pavel@ucw.cz>Peter Cc: Yu-cheng Yu <yu-cheng.yu@intel.com> Subject: [RFC PATCH v4 11/27] x86/mm: Introduce _PAGE_DIRTY_SW Date: Fri, 21 Sep 2018 08:03:35 -0700 [thread overview] Message-ID: <20180921150351.20898-12-yu-cheng.yu@intel.com> (raw) In-Reply-To: <20180921150351.20898-1-yu-cheng.yu@intel.com> A RO and dirty PTE exists in the following cases: (a) A page is modified and then shared with a fork()'ed child; (b) A R/O page that has been COW'ed; (c) A SHSTK page. The processor does not read the dirty bit for (a) and (b), but checks the dirty bit for (c). To prevent the use of non-SHSTK memory as SHSTK, we introduce a spare bit of the 64-bit PTE as _PAGE_BIT_DIRTY_SW and use that for (a) and (b). This results to the following possible PTE settings: Modified PTE: (R/W + DIRTY_HW) Modified and shared PTE: (R/O + DIRTY_SW) R/O PTE COW'ed: (R/O + DIRTY_SW) SHSTK PTE: (R/O + DIRTY_HW) SHSTK PTE COW'ed: (R/O + DIRTY_HW) SHSTK PTE shared: (R/O + DIRTY_SW) Note that _PAGE_BIT_DRITY_SW is only used in R/O PTEs but not R/W PTEs. When this patch is applied, there are six free bits left in the 64-bit PTE. There is no more free bit in the 32-bit PTE (except for PAE) and shadow stack is not implemented for the 32-bit kernel. Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com> --- arch/x86/include/asm/pgtable.h | 129 ++++++++++++++++++++++----- arch/x86/include/asm/pgtable_types.h | 14 ++- 2 files changed, 121 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 95c918ad84ed..3ee554d81480 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -119,9 +119,9 @@ extern pmdval_t early_pmd_flags; * The following only work if pte_present() is true. * Undefined behaviour if not.. */ -static inline int pte_dirty(pte_t pte) +static inline bool pte_dirty(pte_t pte) { - return pte_flags(pte) & _PAGE_DIRTY; + return pte_flags(pte) & _PAGE_DIRTY_BITS; } @@ -143,9 +143,9 @@ static inline int pte_young(pte_t pte) return pte_flags(pte) & _PAGE_ACCESSED; } -static inline int pmd_dirty(pmd_t pmd) +static inline bool pmd_dirty(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_DIRTY; + return pmd_flags(pmd) & _PAGE_DIRTY_BITS; } static inline int pmd_young(pmd_t pmd) @@ -153,9 +153,9 @@ static inline int pmd_young(pmd_t pmd) return pmd_flags(pmd) & _PAGE_ACCESSED; } -static inline int pud_dirty(pud_t pud) +static inline bool pud_dirty(pud_t pud) { - return pud_flags(pud) & _PAGE_DIRTY; + return pud_flags(pud) & _PAGE_DIRTY_BITS; } static inline int pud_young(pud_t pud) @@ -294,9 +294,23 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) return native_make_pte(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + if (pte_flags(pte) & from) + pte = pte_set_flags(pte_clear_flags(pte, from), to); + return pte; +} +#else +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + return pte; +} +#endif + static inline pte_t pte_mkclean(pte_t pte) { - return pte_clear_flags(pte, _PAGE_DIRTY); + return pte_clear_flags(pte, _PAGE_DIRTY_BITS); } static inline pte_t pte_mkold(pte_t pte) @@ -306,6 +320,7 @@ static inline pte_t pte_mkold(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pte_clear_flags(pte, _PAGE_RW); } @@ -316,9 +331,24 @@ static inline pte_t pte_mkexec(pte_t pte) static inline pte_t pte_mkdirty(pte_t pte) { + pteval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHADOW_STACK_USER) || + pte_write(pte)) ? _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pte_set_flags(pte, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pte_t pte_mkdirty_shstk(pte_t pte) +{ + pte = pte_clear_flags(pte, _PAGE_DIRTY_SW); return pte_set_flags(pte, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pte_dirty_hw(pte_t pte) +{ + return pte_flags(pte) & _PAGE_DIRTY_HW; +} +#endif + static inline pte_t pte_mkyoung(pte_t pte) { return pte_set_flags(pte, _PAGE_ACCESSED); @@ -326,6 +356,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkwrite(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pte_set_flags(pte, _PAGE_RW); } @@ -373,6 +404,20 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) return native_make_pmd(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + if (pmd_flags(pmd) & from) + pmd = pmd_set_flags(pmd_clear_flags(pmd, from), to); + return pmd; +} +#else +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + return pmd; +} +#endif + static inline pmd_t pmd_mkold(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_ACCESSED); @@ -380,19 +425,36 @@ static inline pmd_t pmd_mkold(pmd_t pmd) static inline pmd_t pmd_mkclean(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_DIRTY); + return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS); } static inline pmd_t pmd_wrprotect(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pmd_clear_flags(pmd, _PAGE_RW); } static inline pmd_t pmd_mkdirty(pmd_t pmd) { + pmdval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHADOW_STACK_USER) || + (pmd_flags(pmd) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pmd_set_flags(pmd, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pmd_t pmd_mkdirty_shstk(pmd_t pmd) +{ + pmd = pmd_clear_flags(pmd, _PAGE_DIRTY_SW); return pmd_set_flags(pmd, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pmd_dirty_hw(pmd_t pmd) +{ + return pmd_flags(pmd) & _PAGE_DIRTY_HW; +} +#endif + static inline pmd_t pmd_mkdevmap(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_DEVMAP); @@ -410,6 +472,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) static inline pmd_t pmd_mkwrite(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pmd_set_flags(pmd, _PAGE_RW); } @@ -427,6 +490,20 @@ static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) return native_make_pud(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + if (pud_flags(pud) & from) + pud = pud_set_flags(pud_clear_flags(pud, from), to); + return pud; +} +#else +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + return pud; +} +#endif + static inline pud_t pud_mkold(pud_t pud) { return pud_clear_flags(pud, _PAGE_ACCESSED); @@ -434,17 +511,22 @@ static inline pud_t pud_mkold(pud_t pud) static inline pud_t pud_mkclean(pud_t pud) { - return pud_clear_flags(pud, _PAGE_DIRTY); + return pud_clear_flags(pud, _PAGE_DIRTY_BITS); } static inline pud_t pud_wrprotect(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pud_clear_flags(pud, _PAGE_RW); } static inline pud_t pud_mkdirty(pud_t pud) { - return pud_set_flags(pud, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); + pudval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHADOW_STACK_USER) || + (pud_flags(pud) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + + return pud_set_flags(pud, dirty | _PAGE_SOFT_DIRTY); } static inline pud_t pud_mkdevmap(pud_t pud) @@ -464,6 +546,7 @@ static inline pud_t pud_mkyoung(pud_t pud) static inline pud_t pud_mkwrite(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pud_set_flags(pud, _PAGE_RW); } @@ -595,19 +678,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) val &= _PAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK; val = flip_protnone_guard(oldval, val, PTE_PFN_MASK); + if ((pte_write(pte) && !(pgprot_val(newprot) & _PAGE_RW))) + return pte_move_flags(__pte(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); return __pte(val); } -static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) -{ - pmdval_t val = pmd_val(pmd), oldval = val; - - val &= _HPAGE_CHG_MASK; - val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; - val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); - return __pmd(val); -} - /* mprotect needs to preserve PAT bits when updating vm_page_prot */ #define pgprot_modify pgprot_modify static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) @@ -1159,6 +1235,19 @@ static inline int pmd_write(pmd_t pmd) return pmd_flags(pmd) & _PAGE_RW; } +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmdval_t val = pmd_val(pmd), oldval = val; + + val &= _HPAGE_CHG_MASK; + val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; + val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); + if ((pmd_write(pmd) && !(pgprot_val(newprot) & _PAGE_RW))) + return pmd_move_flags(__pmd(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); + return __pmd(val); +} + #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 0657a22d5216..f47bbc1f9c45 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -23,6 +23,7 @@ #define _PAGE_BIT_SOFTW2 10 /* " */ #define _PAGE_BIT_SOFTW3 11 /* " */ #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ +#define _PAGE_BIT_SOFTW5 57 /* available for programmer */ #define _PAGE_BIT_SOFTW4 58 /* available for programmer */ #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ @@ -34,6 +35,7 @@ #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 +#define _PAGE_BIT_DIRTY_SW _PAGE_BIT_SOFTW5 /* was written to */ /* If _PAGE_BIT_PRESENT is clear, we use these: */ /* - if the user mapped it with PROT_NONE; pte_present gives true */ @@ -109,6 +111,14 @@ #define _PAGE_DEVMAP (_AT(pteval_t, 0)) #endif +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +#define _PAGE_DIRTY_SW (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY_SW) +#else +#define _PAGE_DIRTY_SW (_AT(pteval_t, 0)) +#endif + +#define _PAGE_DIRTY_BITS (_PAGE_DIRTY_HW | _PAGE_DIRTY_SW) + #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) #define _PAGE_TABLE_NOENC (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |\ @@ -122,9 +132,9 @@ * instance, and is *not* included in this mask since * pte_modify() does modify it. */ -#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ +#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY_HW | \ - _PAGE_SOFT_DIRTY) + _PAGE_DIRTY_SW | _PAGE_SOFT_DIRTY) #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) /* -- 2.17.1
WARNING: multiple messages have this Message-ID (diff)
From: Yu-cheng Yu <yu-cheng.yu@intel.com> To: x86@kernel.org, "H. Peter Anvin" <hpa@zytor.com>, Thomas Gleixner <tglx@linutronix.de>, Ingo Molnar <mingo@redhat.com>, linux-kernel@vger.kernel.org, linux-doc@vger.kernel.org, linux-mm@kvack.org, linux-arch@vger.kernel.org, linux-api@vger.kernel.org, Arnd Bergmann <arnd@arndb.de>, Andy Lutomirski <luto@amacapital.net>, Balbir Singh <bsingharora@gmail.com>, Cyrill Gorcunov <gorcunov@gmail.com>, Dave Hansen <dave.hansen@linux.intel.com>, Florian Weimer <fweimer@redhat.com>, "H.J. Lu" <hjl.tools@gmail.com>, Jann Horn <jannh@google.com>, Jonathan Corbet <corbet@lwn.net>, Kees Cook <keescook@chromium.org>, Mike Kravetz <mike.kravetz@oracle.com>, Nadav Amit <nadav.amit@gmail.com>, Oleg Nesterov <oleg@redhat.com>, Pavel Machek <pavel@ucw.cz>, Peter Zijlstra <peterz@infradead.org>, Randy Dunlap <rdunlap@infradead.org>, "Ravi V. Shankar" <ravi.v.shankar@intel.com>, Vedvyas Shanbhogue <vedvyas.shanbhogue@intel.com> Cc: Yu-cheng Yu <yu-cheng.yu@intel.com> Subject: [RFC PATCH v4 11/27] x86/mm: Introduce _PAGE_DIRTY_SW Date: Fri, 21 Sep 2018 08:03:35 -0700 [thread overview] Message-ID: <20180921150351.20898-12-yu-cheng.yu@intel.com> (raw) Message-ID: <20180921150335.5B82udl3QN9C5gWynxsOrr8eWoBZYBZh-7RUbkyRzLc@z> (raw) In-Reply-To: <20180921150351.20898-1-yu-cheng.yu@intel.com> A RO and dirty PTE exists in the following cases: (a) A page is modified and then shared with a fork()'ed child; (b) A R/O page that has been COW'ed; (c) A SHSTK page. The processor does not read the dirty bit for (a) and (b), but checks the dirty bit for (c). To prevent the use of non-SHSTK memory as SHSTK, we introduce a spare bit of the 64-bit PTE as _PAGE_BIT_DIRTY_SW and use that for (a) and (b). This results to the following possible PTE settings: Modified PTE: (R/W + DIRTY_HW) Modified and shared PTE: (R/O + DIRTY_SW) R/O PTE COW'ed: (R/O + DIRTY_SW) SHSTK PTE: (R/O + DIRTY_HW) SHSTK PTE COW'ed: (R/O + DIRTY_HW) SHSTK PTE shared: (R/O + DIRTY_SW) Note that _PAGE_BIT_DRITY_SW is only used in R/O PTEs but not R/W PTEs. When this patch is applied, there are six free bits left in the 64-bit PTE. There is no more free bit in the 32-bit PTE (except for PAE) and shadow stack is not implemented for the 32-bit kernel. Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com> --- arch/x86/include/asm/pgtable.h | 129 ++++++++++++++++++++++----- arch/x86/include/asm/pgtable_types.h | 14 ++- 2 files changed, 121 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 95c918ad84ed..3ee554d81480 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -119,9 +119,9 @@ extern pmdval_t early_pmd_flags; * The following only work if pte_present() is true. * Undefined behaviour if not.. */ -static inline int pte_dirty(pte_t pte) +static inline bool pte_dirty(pte_t pte) { - return pte_flags(pte) & _PAGE_DIRTY; + return pte_flags(pte) & _PAGE_DIRTY_BITS; } @@ -143,9 +143,9 @@ static inline int pte_young(pte_t pte) return pte_flags(pte) & _PAGE_ACCESSED; } -static inline int pmd_dirty(pmd_t pmd) +static inline bool pmd_dirty(pmd_t pmd) { - return pmd_flags(pmd) & _PAGE_DIRTY; + return pmd_flags(pmd) & _PAGE_DIRTY_BITS; } static inline int pmd_young(pmd_t pmd) @@ -153,9 +153,9 @@ static inline int pmd_young(pmd_t pmd) return pmd_flags(pmd) & _PAGE_ACCESSED; } -static inline int pud_dirty(pud_t pud) +static inline bool pud_dirty(pud_t pud) { - return pud_flags(pud) & _PAGE_DIRTY; + return pud_flags(pud) & _PAGE_DIRTY_BITS; } static inline int pud_young(pud_t pud) @@ -294,9 +294,23 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) return native_make_pte(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + if (pte_flags(pte) & from) + pte = pte_set_flags(pte_clear_flags(pte, from), to); + return pte; +} +#else +static inline pte_t pte_move_flags(pte_t pte, pteval_t from, pteval_t to) +{ + return pte; +} +#endif + static inline pte_t pte_mkclean(pte_t pte) { - return pte_clear_flags(pte, _PAGE_DIRTY); + return pte_clear_flags(pte, _PAGE_DIRTY_BITS); } static inline pte_t pte_mkold(pte_t pte) @@ -306,6 +320,7 @@ static inline pte_t pte_mkold(pte_t pte) static inline pte_t pte_wrprotect(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pte_clear_flags(pte, _PAGE_RW); } @@ -316,9 +331,24 @@ static inline pte_t pte_mkexec(pte_t pte) static inline pte_t pte_mkdirty(pte_t pte) { + pteval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHADOW_STACK_USER) || + pte_write(pte)) ? _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pte_set_flags(pte, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pte_t pte_mkdirty_shstk(pte_t pte) +{ + pte = pte_clear_flags(pte, _PAGE_DIRTY_SW); return pte_set_flags(pte, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pte_dirty_hw(pte_t pte) +{ + return pte_flags(pte) & _PAGE_DIRTY_HW; +} +#endif + static inline pte_t pte_mkyoung(pte_t pte) { return pte_set_flags(pte, _PAGE_ACCESSED); @@ -326,6 +356,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkwrite(pte_t pte) { + pte = pte_move_flags(pte, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pte_set_flags(pte, _PAGE_RW); } @@ -373,6 +404,20 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) return native_make_pmd(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + if (pmd_flags(pmd) & from) + pmd = pmd_set_flags(pmd_clear_flags(pmd, from), to); + return pmd; +} +#else +static inline pmd_t pmd_move_flags(pmd_t pmd, pmdval_t from, pmdval_t to) +{ + return pmd; +} +#endif + static inline pmd_t pmd_mkold(pmd_t pmd) { return pmd_clear_flags(pmd, _PAGE_ACCESSED); @@ -380,19 +425,36 @@ static inline pmd_t pmd_mkold(pmd_t pmd) static inline pmd_t pmd_mkclean(pmd_t pmd) { - return pmd_clear_flags(pmd, _PAGE_DIRTY); + return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS); } static inline pmd_t pmd_wrprotect(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pmd_clear_flags(pmd, _PAGE_RW); } static inline pmd_t pmd_mkdirty(pmd_t pmd) { + pmdval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHADOW_STACK_USER) || + (pmd_flags(pmd) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + return pmd_set_flags(pmd, dirty | _PAGE_SOFT_DIRTY); +} + +#ifdef CONFIG_ARCH_HAS_SHSTK +static inline pmd_t pmd_mkdirty_shstk(pmd_t pmd) +{ + pmd = pmd_clear_flags(pmd, _PAGE_DIRTY_SW); return pmd_set_flags(pmd, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); } +static inline bool pmd_dirty_hw(pmd_t pmd) +{ + return pmd_flags(pmd) & _PAGE_DIRTY_HW; +} +#endif + static inline pmd_t pmd_mkdevmap(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_DEVMAP); @@ -410,6 +472,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) static inline pmd_t pmd_mkwrite(pmd_t pmd) { + pmd = pmd_move_flags(pmd, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pmd_set_flags(pmd, _PAGE_RW); } @@ -427,6 +490,20 @@ static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) return native_make_pud(v & ~clear); } +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + if (pud_flags(pud) & from) + pud = pud_set_flags(pud_clear_flags(pud, from), to); + return pud; +} +#else +static inline pud_t pud_move_flags(pud_t pud, pudval_t from, pudval_t to) +{ + return pud; +} +#endif + static inline pud_t pud_mkold(pud_t pud) { return pud_clear_flags(pud, _PAGE_ACCESSED); @@ -434,17 +511,22 @@ static inline pud_t pud_mkold(pud_t pud) static inline pud_t pud_mkclean(pud_t pud) { - return pud_clear_flags(pud, _PAGE_DIRTY); + return pud_clear_flags(pud, _PAGE_DIRTY_BITS); } static inline pud_t pud_wrprotect(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_HW, _PAGE_DIRTY_SW); return pud_clear_flags(pud, _PAGE_RW); } static inline pud_t pud_mkdirty(pud_t pud) { - return pud_set_flags(pud, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); + pudval_t dirty = (!IS_ENABLED(CONFIG_X86_INTEL_SHADOW_STACK_USER) || + (pud_flags(pud) & _PAGE_RW)) ? + _PAGE_DIRTY_HW:_PAGE_DIRTY_SW; + + return pud_set_flags(pud, dirty | _PAGE_SOFT_DIRTY); } static inline pud_t pud_mkdevmap(pud_t pud) @@ -464,6 +546,7 @@ static inline pud_t pud_mkyoung(pud_t pud) static inline pud_t pud_mkwrite(pud_t pud) { + pud = pud_move_flags(pud, _PAGE_DIRTY_SW, _PAGE_DIRTY_HW); return pud_set_flags(pud, _PAGE_RW); } @@ -595,19 +678,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) val &= _PAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK; val = flip_protnone_guard(oldval, val, PTE_PFN_MASK); + if ((pte_write(pte) && !(pgprot_val(newprot) & _PAGE_RW))) + return pte_move_flags(__pte(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); return __pte(val); } -static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) -{ - pmdval_t val = pmd_val(pmd), oldval = val; - - val &= _HPAGE_CHG_MASK; - val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; - val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); - return __pmd(val); -} - /* mprotect needs to preserve PAT bits when updating vm_page_prot */ #define pgprot_modify pgprot_modify static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) @@ -1159,6 +1235,19 @@ static inline int pmd_write(pmd_t pmd) return pmd_flags(pmd) & _PAGE_RW; } +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmdval_t val = pmd_val(pmd), oldval = val; + + val &= _HPAGE_CHG_MASK; + val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; + val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); + if ((pmd_write(pmd) && !(pgprot_val(newprot) & _PAGE_RW))) + return pmd_move_flags(__pmd(val), _PAGE_DIRTY_HW, + _PAGE_DIRTY_SW); + return __pmd(val); +} + #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 0657a22d5216..f47bbc1f9c45 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -23,6 +23,7 @@ #define _PAGE_BIT_SOFTW2 10 /* " */ #define _PAGE_BIT_SOFTW3 11 /* " */ #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ +#define _PAGE_BIT_SOFTW5 57 /* available for programmer */ #define _PAGE_BIT_SOFTW4 58 /* available for programmer */ #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ @@ -34,6 +35,7 @@ #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 +#define _PAGE_BIT_DIRTY_SW _PAGE_BIT_SOFTW5 /* was written to */ /* If _PAGE_BIT_PRESENT is clear, we use these: */ /* - if the user mapped it with PROT_NONE; pte_present gives true */ @@ -109,6 +111,14 @@ #define _PAGE_DEVMAP (_AT(pteval_t, 0)) #endif +#if defined(CONFIG_X86_INTEL_SHADOW_STACK_USER) +#define _PAGE_DIRTY_SW (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY_SW) +#else +#define _PAGE_DIRTY_SW (_AT(pteval_t, 0)) +#endif + +#define _PAGE_DIRTY_BITS (_PAGE_DIRTY_HW | _PAGE_DIRTY_SW) + #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) #define _PAGE_TABLE_NOENC (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |\ @@ -122,9 +132,9 @@ * instance, and is *not* included in this mask since * pte_modify() does modify it. */ -#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ +#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY_HW | \ - _PAGE_SOFT_DIRTY) + _PAGE_DIRTY_SW | _PAGE_SOFT_DIRTY) #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) /* -- 2.17.1
next prev parent reply other threads:[~2018-09-21 15:03 UTC|newest] Thread overview: 142+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-09-21 15:03 [RFC PATCH v4 00/27] Control Flow Enforcement: Shadow Stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 01/27] x86/cpufeatures: Add CPUIDs for Control-flow Enforcement Technology (CET) Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-25 16:27 ` Peter Zijlstra 2018-09-25 16:27 ` Peter Zijlstra 2018-09-25 16:29 ` Yu-cheng Yu 2018-09-25 16:29 ` Yu-cheng Yu 2018-09-28 16:51 ` Borislav Petkov 2018-09-28 16:51 ` Borislav Petkov 2018-09-28 16:56 ` Yu-cheng Yu 2018-09-28 16:56 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 02/27] x86/fpu/xstate: Change some names to separate XSAVES system and user states Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-25 16:37 ` Peter Zijlstra 2018-09-25 16:37 ` Peter Zijlstra 2018-10-02 15:29 ` Borislav Petkov 2018-10-02 15:29 ` Borislav Petkov 2018-10-02 16:21 ` Yu-cheng Yu 2018-10-02 16:21 ` Yu-cheng Yu 2018-10-02 16:30 ` Dave Hansen 2018-10-02 16:30 ` Dave Hansen 2018-10-02 16:37 ` Borislav Petkov 2018-10-02 16:37 ` Borislav Petkov 2018-10-02 16:39 ` Dave Hansen 2018-10-02 16:39 ` Dave Hansen 2018-10-02 16:43 ` Yu-cheng Yu 2018-10-02 16:43 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 03/27] x86/fpu/xstate: Enable XSAVES system states Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-25 17:03 ` Peter Zijlstra 2018-09-25 17:03 ` Peter Zijlstra 2018-09-25 17:23 ` Yu-cheng Yu 2018-09-25 17:23 ` Yu-cheng Yu 2018-10-02 17:15 ` Borislav Petkov 2018-10-02 17:15 ` Borislav Petkov 2018-10-04 15:47 ` Yu-cheng Yu 2018-10-04 15:47 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 04/27] x86/fpu/xstate: Add XSAVES system states for shadow stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 05/27] Documentation/x86: Add CET description Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 06/27] x86/cet: Control protection exception handler Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 10:39 ` Eugene Syromiatnikov 2018-10-03 10:39 ` Eugene Syromiatnikov 2018-10-03 16:11 ` Yu-cheng Yu 2018-10-03 16:11 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 07/27] x86/cet/shstk: Add Kconfig option for user-mode shadow stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 08/27] mm: Introduce VM_SHSTK for shadow stack memory Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 09/27] x86/mm: Change _PAGE_DIRTY to _PAGE_DIRTY_HW Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 13:38 ` Matthew Wilcox 2018-10-03 13:38 ` Matthew Wilcox 2018-10-03 14:05 ` Dave Hansen 2018-10-03 14:05 ` Dave Hansen 2018-10-03 16:07 ` Yu-cheng Yu 2018-10-03 16:07 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 10/27] drm/i915/gvt: Update _PAGE_DIRTY to _PAGE_DIRTY_BITS Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 13:19 ` Eugene Syromiatnikov 2018-10-03 13:19 ` Eugene Syromiatnikov 2018-09-21 15:03 ` Yu-cheng Yu [this message] 2018-09-21 15:03 ` [RFC PATCH v4 11/27] x86/mm: Introduce _PAGE_DIRTY_SW Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 12/27] x86/mm: Modify ptep_set_wrprotect and pmdp_set_wrprotect for _PAGE_DIRTY_SW Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 13/27] x86/mm: Shadow stack page fault error checking Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 14/27] mm: Handle shadow stack page fault Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 15/27] mm: Handle THP/HugeTLB " Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 16/27] mm: Update can_follow_write_pte/pmd for shadow stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 17/27] mm: Introduce do_mmap_locked() Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 18/27] x86/cet/shstk: User-mode shadow stack support Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 15:08 ` Eugene Syromiatnikov 2018-10-03 15:08 ` Eugene Syromiatnikov 2018-10-03 15:12 ` Yu-cheng Yu 2018-10-03 15:12 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 19/27] x86/cet/shstk: Introduce WRUSS instruction Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 4:15 ` Eugene Syromiatnikov 2018-10-03 4:15 ` Eugene Syromiatnikov 2018-09-21 15:03 ` [RFC PATCH v4 20/27] x86/cet/shstk: Signal handling for shadow stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 14:36 ` Eugene Syromiatnikov 2018-10-03 14:36 ` Eugene Syromiatnikov 2018-10-03 16:46 ` Jann Horn 2018-10-03 16:46 ` Jann Horn 2018-09-21 15:03 ` [RFC PATCH v4 21/27] x86/cet/shstk: ELF header parsing of Shadow Stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 23:27 ` Eugene Syromiatnikov 2018-10-03 23:27 ` Eugene Syromiatnikov 2018-10-09 21:15 ` Yu-cheng Yu 2018-10-09 21:15 ` Yu-cheng Yu 2018-10-15 23:40 ` Kees Cook 2018-10-15 23:40 ` Kees Cook 2018-10-16 17:23 ` Yu-cheng Yu 2018-10-16 17:23 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 22/27] x86/cet/shstk: Handle thread shadow stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 23/27] mm/map: Add Shadow stack pages to memory accounting Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 16:55 ` Randy Dunlap 2018-09-21 16:55 ` Randy Dunlap 2018-09-21 17:21 ` Yu-cheng Yu 2018-09-21 17:21 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 24/27] mm/mmap: Create a guard area between VMAs Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 4:56 ` Eugene Syromiatnikov 2018-10-03 4:56 ` Eugene Syromiatnikov 2018-10-03 5:36 ` Andy Lutomirski 2018-10-03 5:36 ` Andy Lutomirski 2018-10-03 16:00 ` Yu-cheng Yu 2018-10-03 16:00 ` Yu-cheng Yu 2018-10-03 16:18 ` Andy Lutomirski 2018-10-03 16:18 ` Andy Lutomirski 2018-10-03 16:32 ` Eugene Syromiatnikov 2018-10-03 16:32 ` Eugene Syromiatnikov 2018-10-03 16:40 ` Yu-cheng Yu 2018-10-03 16:40 ` Yu-cheng Yu 2018-10-03 16:52 ` Jann Horn 2018-10-03 16:52 ` Jann Horn 2018-10-03 21:21 ` Eugene Syromiatnikov 2018-10-03 21:21 ` Eugene Syromiatnikov 2018-09-21 15:03 ` [RFC PATCH v4 25/27] mm/mmap: Prevent Shadow Stack VMA merges Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 15:03 ` [RFC PATCH v4 26/27] x86/cet/shstk: Add arch_prctl functions for Shadow Stack Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-10-03 17:57 ` Eugene Syromiatnikov 2018-10-03 17:57 ` Eugene Syromiatnikov 2018-09-21 15:03 ` [RFC PATCH v4 27/27] x86/cet/shstk: Add Shadow Stack instructions to opcode map Yu-cheng Yu 2018-09-21 15:03 ` Yu-cheng Yu 2018-09-21 22:53 ` [RFC PATCH v4 00/27] Control Flow Enforcement: Shadow Stack Dave Hansen 2018-09-21 22:53 ` Dave Hansen 2018-09-24 15:25 ` Yu-cheng Yu 2018-09-24 15:25 ` Yu-cheng Yu
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20180921150351.20898-12-yu-cheng.yu@intel.com \ --to=yu-cheng.yu@intel.com \ --cc=arnd@arndb.de \ --cc=bsingharora@gmail.com \ --cc=corbet@lwn.net \ --cc=dave.hansen@linux.intel.com \ --cc=fweimer@redhat.com \ --cc=gorcunov@gmail.com \ --cc=hjl.tools@gmail.com \ --cc=hpa@zytor.com \ --cc=jannh@google.com \ --cc=keescook@chromium.org \ --cc=linux-api@vger.kernel.org \ --cc=linux-arch@vger.kernel.org \ --cc=linux-doc@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=luto@amacapital.net \ --cc=mike.kravetz@oracle.com \ --cc=mingo@redhat.com \ --cc=nadav.amit@gmail.com \ --cc=oleg@redhat.com \ --cc=pavel@ucw.cz \ --cc=tglx@linutronix.de \ --cc=x86@kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).