From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753291AbdLDRCn (ORCPT ); Mon, 4 Dec 2017 12:02:43 -0500 Received: from Galois.linutronix.de ([146.0.238.70]:60080 "EHLO Galois.linutronix.de" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752350AbdLDQvU (ORCPT ); Mon, 4 Dec 2017 11:51:20 -0500 Message-Id: <20171204150607.391576490@linutronix.de> User-Agent: quilt/0.63-1 Date: Mon, 04 Dec 2017 15:07:37 +0100 From: Thomas Gleixner To: LKML Cc: x86@kernel.org, Linus Torvalds , Andy Lutomirsky , Peter Zijlstra , Dave Hansen , Borislav Petkov , Greg KH , keescook@google.com, hughd@google.com, Brian Gerst , Josh Poimboeuf , Denys Vlasenko , Rik van Riel , Boris Ostrovsky , Juergen Gross , David Laight , Eduardo Valentin , aliguori@amazon.com, Will Deacon , daniel.gruss@iaik.tugraz.at, Dave Hansen Subject: [patch 31/60] x86/mm/kpti: Add mapping helper functions References: <20171204140706.296109558@linutronix.de> MIME-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-15 Content-Disposition: inline; filename=x86-kpti--Add_mapping_helper_functions.patch Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org From: Dave Hansen Add the pagetable helper functions do manage the separate user space page tables. [ tglx: Split out from the big combo kaiser patch ] Signed-off-by: Dave Hansen Signed-off-by: Thomas Gleixner --- arch/x86/include/asm/pgtable_64.h | 139 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -131,9 +131,144 @@ static inline pud_t native_pudp_get_and_ #endif } +#ifdef CONFIG_KERNEL_PAGE_TABLE_ISOLATION +/* + * All top-level KERNEL_PAGE_TABLE_ISOLATION page tables are order-1 pages + * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and + * the user one is in the last 4k. To switch between them, you + * just need to flip the 12th bit in their addresses. + */ +#define KPTI_PGTABLE_SWITCH_BIT PAGE_SHIFT + +/* + * This generates better code than the inline assembly in + * __set_bit(). + */ +static inline void *ptr_set_bit(void *ptr, int bit) +{ + unsigned long __ptr = (unsigned long)ptr; + + __ptr |= BIT(bit); + return (void *)__ptr; +} +static inline void *ptr_clear_bit(void *ptr, int bit) +{ + unsigned long __ptr = (unsigned long)ptr; + + __ptr &= ~BIT(bit); + return (void *)__ptr; +} + +static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) +{ + return ptr_set_bit(pgdp, KPTI_PGTABLE_SWITCH_BIT); +} + +static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) +{ + return ptr_clear_bit(pgdp, KPTI_PGTABLE_SWITCH_BIT); +} + +static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) +{ + return ptr_set_bit(p4dp, KPTI_PGTABLE_SWITCH_BIT); +} + +static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) +{ + return ptr_clear_bit(p4dp, KPTI_PGTABLE_SWITCH_BIT); +} +#endif /* CONFIG_KERNEL_PAGE_TABLE_ISOLATION */ + +/* + * Page table pages are page-aligned. The lower half of the top + * level is used for userspace and the top half for the kernel. + * + * Returns true for parts of the PGD that map userspace and + * false for the parts that map the kernel. + */ +static inline bool pgdp_maps_userspace(void *__ptr) +{ + unsigned long ptr = (unsigned long)__ptr; + + return (ptr & ~PAGE_MASK) < (PAGE_SIZE / 2); +} + +/* + * Does this PGD allow access from userspace? + */ +static inline bool pgd_userspace_access(pgd_t pgd) +{ + return pgd.pgd & _PAGE_USER; +} + +/* + * Take a PGD location (pgdp) and a pgd value that needs to be set there. + * Populates the user and returns the resulting PGD that must be set in + * the kernel copy of the page tables. + */ +static inline pgd_t kpti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) +{ +#ifdef CONFIG_KERNEL_PAGE_TABLE_ISOLATION + if (!static_cpu_has_bug(X86_BUG_CPU_SECURE_MODE_KPTI)) + return pgd; + + if (pgd_userspace_access(pgd)) { + if (pgdp_maps_userspace(pgdp)) { + /* + * The user page tables get the full PGD, + * accessible from userspace: + */ + kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; + /* + * For the copy of the pgd that the kernel uses, + * make it unusable to userspace. This ensures on + * in case that a return to userspace with the + * kernel CR3 value, userspace will crash instead + * of running. + * + * Note: NX might be not available or disabled. + */ + if (__supported_pte_mask & _PAGE_NX) + pgd.pgd |= _PAGE_NX; + } + } else if (pgd_userspace_access(*pgdp)) { + /* + * We are clearing a _PAGE_USER PGD for which we presumably + * populated the user PGD. We must now clear the user PGD + * entry. + */ + if (pgdp_maps_userspace(pgdp)) { + kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; + } else { + /* + * Attempted to clear a _PAGE_USER PGD which is in + * the kernel porttion of the address space. PGDs + * are pre-populated and we never clear them. + */ + WARN_ON_ONCE(1); + } + } else { + /* + * _PAGE_USER was not set in either the PGD being set or + * cleared. All kernel PGDs should be pre-populated so + * this should never happen after boot. + */ + WARN_ON_ONCE(system_state == SYSTEM_RUNNING); + } +#endif + /* return the copy of the PGD we want the kernel to use: */ + return pgd; +} + + static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) { +#if defined(CONFIG_KERNEL_PAGE_TABLE_ISOLATION) && !defined(CONFIG_X86_5LEVEL) + p4dp->pgd = kpti_set_user_pgd(&p4dp->pgd, p4d.pgd); +#else *p4dp = p4d; +#endif } static inline void native_p4d_clear(p4d_t *p4d) @@ -147,7 +282,11 @@ static inline void native_p4d_clear(p4d_ static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) { +#ifdef CONFIG_KERNEL_PAGE_TABLE_ISOLATION + *pgdp = kpti_set_user_pgd(pgdp, pgd); +#else *pgdp = pgd; +#endif } static inline void native_pgd_clear(pgd_t *pgd)