kvm.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Joerg Roedel <joro@8bytes.org>
To: x86@kernel.org
Cc: Joerg Roedel <joro@8bytes.org>, Joerg Roedel <jroedel@suse.de>,
	hpa@zytor.com, Andy Lutomirski <luto@kernel.org>,
	Dave Hansen <dave.hansen@linux.intel.com>,
	Peter Zijlstra <peterz@infradead.org>,
	Jiri Slaby <jslaby@suse.cz>,
	Dan Williams <dan.j.williams@intel.com>,
	Tom Lendacky <thomas.lendacky@amd.com>,
	Juergen Gross <jgross@suse.com>,
	Kees Cook <keescook@chromium.org>,
	David Rientjes <rientjes@google.com>,
	Cfir Cohen <cfir@google.com>, Erdem Aktas <erdemaktas@google.com>,
	Masami Hiramatsu <mhiramat@kernel.org>,
	Mike Stunes <mstunes@vmware.com>,
	Sean Christopherson <sean.j.christopherson@intel.com>,
	Martin Radev <martin.b.radev@gmail.com>,
	linux-kernel@vger.kernel.org, kvm@vger.kernel.org,
	virtualization@lists.linux-foundation.org
Subject: [PATCH v5 21/75] x86/boot/compressed/64: Add set_page_en/decrypted() helpers
Date: Fri, 24 Jul 2020 18:02:42 +0200	[thread overview]
Message-ID: <20200724160336.5435-22-joro@8bytes.org> (raw)
In-Reply-To: <20200724160336.5435-1-joro@8bytes.org>

From: Joerg Roedel <jroedel@suse.de>

The functions are needed to map the GHCB for SEV-ES guests. The GHCB is
used for communication with the hypervisor, so its content must not be
encrypted. After the GHCB is not needed anymore it must be mapped
encrypted again so that the running kernel image can safely re-use the
memory.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
---
 arch/x86/boot/compressed/ident_map_64.c | 133 ++++++++++++++++++++++++
 arch/x86/boot/compressed/misc.h         |   2 +
 2 files changed, 135 insertions(+)

diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c
index aa91bebc0fe9..05742f641a06 100644
--- a/arch/x86/boot/compressed/ident_map_64.c
+++ b/arch/x86/boot/compressed/ident_map_64.c
@@ -24,6 +24,7 @@
 
 /* These actually do the work of building the kernel identity maps. */
 #include <linux/pgtable.h>
+#include <asm/cmpxchg.h>
 #include <asm/trap_pf.h>
 #include <asm/trapnr.h>
 #include <asm/init.h>
@@ -165,6 +166,138 @@ void finalize_identity_maps(void)
 	write_cr3(top_level_pgt);
 }
 
+static pte_t *split_large_pmd(struct x86_mapping_info *info,
+			      pmd_t *pmdp, unsigned long __address)
+{
+	unsigned long page_flags;
+	unsigned long address;
+	pte_t *pte;
+	pmd_t pmd;
+	int i;
+
+	pte = (pte_t *)info->alloc_pgt_page(info->context);
+	if (!pte)
+		return NULL;
+
+	address     = __address & PMD_MASK;
+	/* No large page - clear PSE flag */
+	page_flags  = info->page_flag & ~_PAGE_PSE;
+
+	/* Populate the PTEs */
+	for (i = 0; i < PTRS_PER_PMD; i++) {
+		set_pte(&pte[i], __pte(address | page_flags));
+		address += PAGE_SIZE;
+	}
+
+	/*
+	 * Ideally we need to clear the large PMD first and do a TLB
+	 * flush before we write the new PMD. But the 2M range of the
+	 * PMD might contain the code we execute and/or the stack
+	 * we are on, so we can't do that. But that should be safe here
+	 * because we are going from large to small mappings and we are
+	 * also the only user of the page-table, so there is no chance
+	 * of a TLB multihit.
+	 */
+	pmd = __pmd((unsigned long)pte | info->kernpg_flag);
+	set_pmd(pmdp, pmd);
+	/* Flush TLB to establish the new PMD */
+	write_cr3(top_level_pgt);
+
+	return pte + pte_index(__address);
+}
+
+static void clflush_page(unsigned long address)
+{
+	unsigned int flush_size;
+	char *cl, *start, *end;
+
+	/*
+	 * Hardcode cl-size to 64 - CPUID can't be used here because that might
+	 * cause another #VC exception and the GHCB is not ready to use yet.
+	 */
+	flush_size = 64;
+	start      = (char *)(address & PAGE_MASK);
+	end        = start + PAGE_SIZE;
+
+	/*
+	 * First make sure there are no pending writes on the cache-lines to
+	 * flush.
+	 */
+	asm volatile("mfence" : : : "memory");
+
+	for (cl = start; cl != end; cl += flush_size)
+		clflush(cl);
+}
+
+static int set_clr_page_flags(struct x86_mapping_info *info,
+			      unsigned long address,
+			      pteval_t set, pteval_t clr)
+{
+	pgd_t *pgdp = (pgd_t *)top_level_pgt;
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep, pte;
+
+	/*
+	 * First make sure there is a PMD mapping for 'address'.
+	 * It should already exist, but keep things generic.
+	 *
+	 * To map the page just read from it and fault it in if there is no
+	 * mapping yet. add_identity_map() can't be called here because that
+	 * would unconditionally map the address on PMD level, destroying any
+	 * PTE-level mappings that might already exist. Use assembly here so
+	 * the access won't be optimized away.
+	 */
+	asm volatile("mov %[address], %%r9"
+		     :: [address] "g" (*(unsigned long *)address)
+		     : "r9", "memory");
+
+	/*
+	 * The page is mapped at least with PMD size - so skip checks and walk
+	 * directly to the PMD.
+	 */
+	p4dp = p4d_offset(pgdp, address);
+	pudp = pud_offset(p4dp, address);
+	pmdp = pmd_offset(pudp, address);
+
+	if (pmd_large(*pmdp))
+		ptep = split_large_pmd(info, pmdp, address);
+	else
+		ptep = pte_offset_kernel(pmdp, address);
+
+	if (!ptep)
+		return -ENOMEM;
+
+	/*
+	 * Changing encryption attributes of a page requires to flush it from
+	 * the caches.
+	 */
+	if ((set | clr) & _PAGE_ENC)
+		clflush_page(address);
+
+	/* Update PTE */
+	pte = *ptep;
+	pte = pte_set_flags(pte, set);
+	pte = pte_clear_flags(pte, clr);
+	set_pte(ptep, pte);
+
+	/* Flush TLB after changing encryption attribute */
+	write_cr3(top_level_pgt);
+
+	return 0;
+}
+
+int set_page_decrypted(unsigned long address)
+{
+	return set_clr_page_flags(&mapping_info, address, 0, _PAGE_ENC);
+}
+
+int set_page_encrypted(unsigned long address)
+{
+	return set_clr_page_flags(&mapping_info, address, _PAGE_ENC, 0);
+}
+
 static void do_pf_error(const char *msg, unsigned long error_code,
 			unsigned long address, unsigned long ip)
 {
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index 65da40777bc1..5e569e8a7d75 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -98,6 +98,8 @@ static inline void choose_random_location(unsigned long input,
 #endif
 
 #ifdef CONFIG_X86_64
+extern int set_page_decrypted(unsigned long address);
+extern int set_page_encrypted(unsigned long address);
 extern unsigned char _pgtable[];
 #endif
 
-- 
2.27.0


  parent reply	other threads:[~2020-07-24 16:10 UTC|newest]

Thread overview: 97+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-07-24 16:02 [PATCH v5 00/75] x86: SEV-ES Guest Support Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 01/75] KVM: SVM: Add GHCB definitions Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 02/75] KVM: SVM: Add GHCB Accessor functions Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 03/75] KVM: SVM: Use __packed shorthand Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 04/75] x86/cpufeatures: Add SEV-ES CPU feature Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 05/75] x86/traps: Move pf error codes to <asm/trap_pf.h> Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 06/75] x86/insn: Make inat-tables.c suitable for pre-decompression code Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 07/75] x86/umip: Factor out instruction fetch Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 08/75] x86/umip: Factor out instruction decoding Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 09/75] x86/insn: Add insn_get_modrm_reg_off() Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 10/75] x86/insn: Add insn_has_rep_prefix() helper Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 11/75] x86/boot/compressed/64: Disable red-zone usage Joerg Roedel
2020-07-24 17:43   ` Kees Cook
2020-07-24 17:58   ` Arvind Sankar
2020-07-24 16:02 ` [PATCH v5 12/75] x86/boot/compressed/64: Add IDT Infrastructure Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 13/75] x86/boot/compressed/64: Rename kaslr_64.c to ident_map_64.c Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 14/75] x86/boot/compressed/64: Add page-fault handler Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 15/75] x86/boot/compressed/64: Always switch to own page-table Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 16/75] x86/boot/compressed/64: Don't pre-map memory in KASLR code Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 17/75] x86/boot/compressed/64: Change add_identity_map() to take start and end Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 18/75] x86/boot/compressed/64: Add stage1 #VC handler Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 19/75] x86/boot/compressed/64: Call set_sev_encryption_mask earlier Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 20/75] x86/boot/compressed/64: Check return value of kernel_ident_mapping_init() Joerg Roedel
2020-07-24 16:02 ` Joerg Roedel [this message]
2020-07-24 16:02 ` [PATCH v5 22/75] x86/boot/compressed/64: Setup GHCB Based VC Exception handler Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 23/75] x86/boot/compressed/64: Unmap GHCB page before booting the kernel Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 24/75] x86/sev-es: Add support for handling IOIO exceptions Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 25/75] x86/fpu: Move xgetbv()/xsetbv() into separate header Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 26/75] x86/sev-es: Add CPUID handling to #VC handler Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 27/75] x86/idt: Move IDT to data segment Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 28/75] x86/idt: Split idt_data setup out of set_intr_gate() Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 29/75] x86/head/64: Install startup GDT Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 30/75] x86/head/64: Setup MSR_GS_BASE before calling into C code Joerg Roedel
2020-07-24 17:42   ` Kees Cook
2020-07-24 16:02 ` [PATCH v5 31/75] x86/head/64: Load GDT after switch to virtual addresses Joerg Roedel
2020-07-24 17:40   ` Kees Cook
2020-07-24 16:02 ` [PATCH v5 32/75] x86/head/64: Load segment registers earlier Joerg Roedel
2020-07-24 17:42   ` Kees Cook
2020-07-24 16:02 ` [PATCH v5 33/75] x86/head/64: Switch to initial stack earlier Joerg Roedel
2020-07-24 17:43   ` Kees Cook
2020-07-24 16:02 ` [PATCH v5 34/75] x86/head/64: Make fixup_pointer() static inline Joerg Roedel
2020-07-24 17:52   ` Kees Cook
2020-07-24 16:02 ` [PATCH v5 35/75] x86/head/64: Load IDT earlier Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 36/75] x86/head/64: Move early exception dispatch to C code Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 37/75] x86/head/64: Set CR4.FSGSBASE early Joerg Roedel
2020-07-24 16:02 ` [PATCH v5 38/75] x86/sev-es: Add SEV-ES Feature Detection Joerg Roedel
2020-07-24 17:54   ` Kees Cook
2020-07-24 16:03 ` [PATCH v5 39/75] x86/sev-es: Print SEV-ES info into kernel log Joerg Roedel
2020-07-24 17:54   ` Kees Cook
2020-07-24 16:03 ` [PATCH v5 40/75] x86/sev-es: Compile early handler code into kernel image Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 41/75] x86/sev-es: Setup early #VC handler Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 42/75] x86/sev-es: Setup GHCB based boot " Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 43/75] x86/sev-es: Setup per-cpu GHCBs for the runtime handler Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 44/75] x86/sev-es: Allocate and Map IST stack for #VC handler Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 45/75] x86/sev-es: Adjust #VC IST Stack on entering NMI handler Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 46/75] x86/dumpstack/64: Add noinstr version of get_stack_info() Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 47/75] x86/entry/64: Add entry code for #VC handler Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 48/75] x86/sev-es: Add Runtime #VC Exception Handler Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 49/75] x86/sev-es: Wire up existing #VC exit-code handlers Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 50/75] x86/sev-es: Handle instruction fetches from user-space Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 51/75] x86/sev-es: Handle MMIO events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 52/75] x86/sev-es: Handle MMIO String Instructions Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 53/75] x86/sev-es: Handle MSR events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 54/75] x86/sev-es: Handle DR7 read/write events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 55/75] x86/sev-es: Handle WBINVD Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 56/75] x86/sev-es: Handle RDTSC(P) Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 57/75] x86/sev-es: Handle RDPMC Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 58/75] x86/sev-es: Handle INVD Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 59/75] x86/sev-es: Handle MONITOR/MONITORX Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 60/75] x86/sev-es: Handle MWAIT/MWAITX Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 61/75] x86/sev-es: Handle VMMCALL Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 62/75] x86/sev-es: Handle #AC Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 63/75] x86/sev-es: Handle #DB Events Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 64/75] x86/paravirt: Allow hypervisor specific VMMCALL handling under SEV-ES Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 65/75] x86/kvm: Add KVM " Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 66/75] x86/vmware: Add VMware specific handling for VMMCALL " Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 67/75] x86/realmode: Add SEV-ES specific trampoline entry point Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 68/75] x86/realmode: Setup AP jump table Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 69/75] x86/smpboot: Setup TSS for starting AP Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 70/75] x86/head/64: Don't call verify_cpu() on starting APs Joerg Roedel
2020-07-24 17:57   ` Kees Cook
2020-07-24 16:03 ` [PATCH v5 71/75] x86/head/64: Rename start_cpu0 Joerg Roedel
2020-07-24 17:56   ` Kees Cook
2020-07-24 16:03 ` [PATCH v5 72/75] x86/sev-es: Support CPU offline/online Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 73/75] x86/sev-es: Handle NMI State Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 74/75] x86/efi: Add GHCB mappings when SEV-ES is active Joerg Roedel
2020-07-24 16:03 ` [PATCH v5 75/75] x86/sev-es: Check required CPU features for SEV-ES Joerg Roedel
2020-07-24 17:55   ` Kees Cook
2020-07-30  1:27 ` [PATCH v5 00/75] x86: SEV-ES Guest Support Mike Stunes
2020-07-30 12:26   ` Joerg Roedel
2020-07-30 23:23     ` Mike Stunes
2020-08-18 15:07       ` Joerg Roedel
2020-08-20  0:58         ` Mike Stunes
2020-08-20 12:10           ` Joerg Roedel
2020-08-21  8:05           ` Joerg Roedel
2020-08-21 17:42             ` Mike Stunes
2020-08-22 16:30               ` Joerg Roedel

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200724160336.5435-22-joro@8bytes.org \
    --to=joro@8bytes.org \
    --cc=cfir@google.com \
    --cc=dan.j.williams@intel.com \
    --cc=dave.hansen@linux.intel.com \
    --cc=erdemaktas@google.com \
    --cc=hpa@zytor.com \
    --cc=jgross@suse.com \
    --cc=jroedel@suse.de \
    --cc=jslaby@suse.cz \
    --cc=keescook@chromium.org \
    --cc=kvm@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=luto@kernel.org \
    --cc=martin.b.radev@gmail.com \
    --cc=mhiramat@kernel.org \
    --cc=mstunes@vmware.com \
    --cc=peterz@infradead.org \
    --cc=rientjes@google.com \
    --cc=sean.j.christopherson@intel.com \
    --cc=thomas.lendacky@amd.com \
    --cc=virtualization@lists.linux-foundation.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).