From: Tom Lendacky <thomas.lendacky@amd.com> To: linux-arch@vger.kernel.org, linux-efi@vger.kernel.org, kvm@vger.kernel.org, linux-doc@vger.kernel.org, x86@kernel.org, linux-kernel@vger.kernel.org, kasan-dev@googlegroups.com, linux-mm@kvack.org, iommu@lists.linux-foundation.org Cc: "Radim Krčmář" <rkrcmar@redhat.com>, "Arnd Bergmann" <arnd@arndb.de>, "Jonathan Corbet" <corbet@lwn.net>, "Matt Fleming" <matt@codeblueprint.co.uk>, "Joerg Roedel" <joro@8bytes.org>, "Konrad Rzeszutek Wilk" <konrad.wilk@oracle.com>, "Paolo Bonzini" <pbonzini@redhat.com>, "Ingo Molnar" <mingo@redhat.com>, "Borislav Petkov" <bp@alien8.de>, "H. Peter Anvin" <hpa@zytor.com>, "Andrey Ryabinin" <aryabinin@virtuozzo.com>, "Alexander Potapenko" <glider@google.com>, "Thomas Gleixner" <tglx@linutronix.de>, "Dmitry Vyukov" <dvyukov@google.com> Subject: [RFC PATCH v1 18/18] x86: Add support to turn on Secure Memory Encryption Date: Tue, 26 Apr 2016 17:59:04 -0500 [thread overview] Message-ID: <20160426225904.13567.538.stgit@tlendack-t1.amdoffice.net> (raw) In-Reply-To: <20160426225553.13567.19459.stgit@tlendack-t1.amdoffice.net> This patch adds the support to check for and enable SME when available on the processor and when the mem_encrypt=on command line option is set. This consists of setting the encryption mask, calculating the number of physical bits of addressing lost and encrypting the kernel "in place." Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com> --- Documentation/kernel-parameters.txt | 3 arch/x86/kernel/asm-offsets.c | 2 arch/x86/kernel/mem_encrypt.S | 306 +++++++++++++++++++++++++++++++++++ 3 files changed, 311 insertions(+) diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 8ba7f82..0a2678a 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -2210,6 +2210,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted. memory contents and reserves bad memory regions that are detected. + mem_encrypt=on [X86_64] Enable memory encryption on processors + that support this feature. + meye.*= [HW] Set MotionEye Camera parameters See Documentation/video4linux/meye.txt. diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 5c04246..a0f76de 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -82,6 +82,8 @@ void common(void) { OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment); OFFSET(BP_pref_address, boot_params, hdr.pref_address); OFFSET(BP_code32_start, boot_params, hdr.code32_start); + OFFSET(BP_cmd_line_ptr, boot_params, hdr.cmd_line_ptr); + OFFSET(BP_ext_cmd_line_ptr, boot_params, ext_cmd_line_ptr); BLANK(); DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); diff --git a/arch/x86/kernel/mem_encrypt.S b/arch/x86/kernel/mem_encrypt.S index f2e0536..4d3326d 100644 --- a/arch/x86/kernel/mem_encrypt.S +++ b/arch/x86/kernel/mem_encrypt.S @@ -12,13 +12,236 @@ #include <linux/linkage.h> +#include <asm/processor-flags.h> +#include <asm/pgtable.h> +#include <asm/page.h> +#include <asm/msr.h> +#include <asm/asm-offsets.h> + .text .code64 ENTRY(sme_enable) +#ifdef CONFIG_AMD_MEM_ENCRYPT + /* Check for AMD processor */ + xorl %eax, %eax + cpuid + cmpl $0x68747541, %ebx # AuthenticAMD + jne .Lno_mem_encrypt + cmpl $0x69746e65, %edx + jne .Lno_mem_encrypt + cmpl $0x444d4163, %ecx + jne .Lno_mem_encrypt + + /* Check for memory encryption leaf */ + movl $0x80000000, %eax + cpuid + cmpl $0x8000001f, %eax + jb .Lno_mem_encrypt + + /* + * Check for memory encryption feature: + * CPUID Fn8000_001F[EAX] - Bit 0 + */ + movl $0x8000001f, %eax + cpuid + bt $0, %eax + jnc .Lno_mem_encrypt + + /* Check for the mem_encrypt=on command line option */ + push %rsi /* Save RSI (real_mode_data) */ + movl BP_ext_cmd_line_ptr(%rsi), %ecx + shlq $32, %rcx + movl BP_cmd_line_ptr(%rsi), %edi + addq %rcx, %rdi + leaq mem_encrypt_enable_option(%rip), %rsi + call cmdline_find_option_bool + pop %rsi /* Restore RSI (real_mode_data) */ + testl %eax, %eax + jz .Lno_mem_encrypt + + /* + * Get memory encryption information: + * CPUID Fn8000_001F[EBX] - Bits 5:0 + * Pagetable bit position used to indicate encryption + */ + movl %ebx, %ecx + andl $0x3f, %ecx + jz .Lno_mem_encrypt + bts %ecx, sme_me_mask(%rip) + shrl $6, %ebx + + /* + * Get memory encryption information: + * CPUID Fn8000_001F[EBX] - Bits 11:6 + * Reduction in physical address space (in bits) when enabled + */ + movl %ebx, %ecx + andl $0x3f, %ecx + movb %cl, sme_me_loss(%rip) + + /* + * Enable memory encryption through the SYSCFG MSR + */ + movl $MSR_K8_SYSCFG, %ecx + rdmsr + bt $MSR_K8_SYSCFG_MEM_ENCRYPT_BIT, %eax + jc .Lmem_encrypt_exit + bts $MSR_K8_SYSCFG_MEM_ENCRYPT_BIT, %eax + wrmsr + jmp .Lmem_encrypt_exit + +.Lno_mem_encrypt: + /* Did not get enabled, clear settings */ + movq $0, sme_me_mask(%rip) + movb $0, sme_me_loss(%rip) + +.Lmem_encrypt_exit: +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + ret ENDPROC(sme_enable) ENTRY(sme_encrypt_kernel) +#ifdef CONFIG_AMD_MEM_ENCRYPT + cmpq $0, sme_me_mask(%rip) + jz .Lencrypt_exit + + /* + * Encrypt the kernel. + * Pagetables for performing kernel encryption: + * 0x0000000000 - 0x00FFFFFFFF will map just the memory occupied by + * the kernel as encrypted memory + * 0x8000000000 - 0x80FFFFFFFF will map all memory as write-protected, + * non-encrypted + * + * The use of write-protected memory will prevent any of the + * non-encrypted memory from being cached. + * + * 0x00... and 0x80... represent the first and second PGD entries. + * + * This collection of entries will be created in an area outside + * of the area that is being encrypted (outside the kernel) and + * requires 11 4K pages: + * 1 - PGD + * 2 - PUDs (1 for each mapping) + * 8 - PMDs (4 for each mapping) + */ + leaq _end(%rip), %rdi + addq $~PMD_PAGE_MASK, %rdi + andq $PMD_PAGE_MASK, %rdi /* RDI points to the new PGD */ + + /* Clear the pagetable memory */ + movq %rdi, %rbx /* Save pointer to PGD */ + movl $(4096 * 11), %ecx + xorl %eax, %eax + rep stosb + movq %rbx, %rdi /* Restore pointer to PGD */ + + /* Set up PGD entries for the two mappings */ + leaq (0x1000 + 0x03)(%rdi), %rbx /* PUD for encrypted kernel */ + movq %rbx, (%rdi) + leaq (0x2000 + 0x03)(%rdi), %rbx /* PUD for unencrypted kernel */ + movq %rbx, 8(%rdi) + + /* Set up PUD entries (4 per mapping) for the two mappings */ + leaq (0x3000 + 0x03)(%rdi), %rbx /* PMD for encrypted kernel */ + leaq (0x7000 + 0x03)(%rdi), %rdx /* PMD for unencrypted kernel */ + xorq %rcx, %rcx +1: + /* Populate the PUD entries in each mapping */ + movq %rbx, 0x1000(%rdi, %rcx, 8) + movq %rdx, 0x2000(%rdi, %rcx, 8) + addq $0x1000, %rbx + addq $0x1000, %rdx + incq %rcx + cmpq $4, %rcx + jb 1b + + /* + * Set up PMD entries (4GB worth) for the two mappings. + * For the encrypted kernel mapping, when R11 is above RDX + * and below RDI then we know we are in the kernel and we + * set the encryption mask for that PMD entry. + * + * The use of _PAGE_PAT and _PAGE_PWT will provide for the + * write-protected mapping. + */ + movq sme_me_mask(%rip), %r10 + movq $__PAGE_KERNEL_LARGE_EXEC, %r11 + andq $~_PAGE_GLOBAL, %r11 + movq %r11, %r12 + andq $~_PAGE_CACHE_MASK, %r12 + orq $(_PAGE_PAT | _PAGE_PWT), %r12 /* PA5 index */ + xorq %rcx, %rcx + leaq _text(%rip), %rdx /* RDX points to start of kernel */ +1: + /* Populate the PMD entries in each mapping */ + movq %r11, 0x3000(%rdi, %rcx, 8) + movq %r12, 0x7000(%rdi, %rcx, 8) + + /* + * Check if we are in the kernel range, and if so, set the + * memory encryption mask. + */ + cmpq %r11, %rdx + jae 2f + cmpq %r11, %rdi + jbe 2f + orq %r10, 0x3000(%rdi, %rcx, 8) +2: + addq $PMD_SIZE, %r11 + addq $PMD_SIZE, %r12 + incq %rcx + cmpq $2048, %rcx + jb 1b + + /* + * Set up a one page stack in the non-encrypted memory area. + * Set RAX to point to the next page in memory after all the + * page tables. The stack grows from the bottom so point to + * the end of the page. + */ + leaq (4096 * 11)(%rdi), %rax + addq $PAGE_SIZE, %rax + movq %rsp, %rbp + movq %rax, %rsp + push %rbp /* Save original stack pointer */ + + push %rsi /* Save RSI (real mode data) */ + + /* + * Copy encryption routine into safe memory + * - RAX points to the page after all the page tables and stack + * where the routine will copied + * - RDI points to the PGD table + * - Setup registers for call + * and then call it + */ + movq %rdi, %rbx + + leaq .Lencrypt_start(%rip), %rsi + movq %rax, %rdi + movq $(.Lencrypt_stop - .Lencrypt_start), %rcx + rep movsb + + leaq _text(%rip), %rsi /* Kernel start */ + movq %rbx, %rcx /* New PGD start */ + subq %rsi, %rcx /* Size of area to encrypt */ + + movq %rsi, %rdi /* Encrypted kernel space start */ + movq $1, %rsi + shlq $PGDIR_SHIFT, %rsi + addq %rdi, %rsi /* Non-encrypted kernel start */ + + /* Call the encryption routine */ + call *%rax + + pop %rsi /* Restore RSI (real mode data ) */ + + pop %rsp /* Restore original stack pointer */ +.Lencrypt_exit: +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + ret ENDPROC(sme_encrypt_kernel) @@ -28,6 +251,87 @@ ENTRY(sme_get_me_loss) ret ENDPROC(sme_get_me_loss) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * Routine used to encrypt kernel. + * This routine must be run outside of the kernel proper since + * the kernel will be encrypted during the process. So this + * routine is defined here and then copied to an area outside + * of the kernel where it will remain and run un-encrypted + * during execution. + * + * On entry the registers must be: + * - RAX points to this routine + * - RBX points to new PGD to use + * - RCX contains the kernel length + * - RSI points to the non-encrypted kernel space + * - RDI points to the encrypted kernel space + * + * The kernel will be encrypted by copying from the non-encrypted + * kernel space to a temporary buffer and then copying from the + * temporary buffer back to the encrypted kernel space. The physical + * addresses of the two kernel space mappings are the same which + * results in the kernel being encrypted "in place". + */ +.Lencrypt_start: + /* Enable the new page tables */ + mov %rbx, %cr3 + + /* Flush any global TLBs */ + mov %cr4, %rbx + andq $~X86_CR4_PGE, %rbx + mov %rbx, %cr4 + orq $X86_CR4_PGE, %rbx + mov %rbx, %cr4 + + /* Set the PAT register PA5 entry to write-protect */ + push %rax + push %rcx + movl $MSR_IA32_CR_PAT, %ecx + rdmsr + push %rdx /* Save original PAT value */ + andl $0xffff00ff, %edx /* Clear PA5 */ + orl $0x00000500, %edx /* Set PA5 to WP */ + wrmsr + pop %rdx /* RDX contains original PAT value */ + pop %rcx + pop %rax + + movq %rsi, %r10 /* Save source address */ + movq %rdi, %r11 /* Save destination address */ + movq %rcx, %r12 /* Save length */ + addq $PAGE_SIZE, %rax /* RAX now points to temp copy page */ + + wbinvd /* Invalidate any cache entries */ + + /* Copy/encrypt 2MB at a time */ +1: + movq %r10, %rsi + movq %rax, %rdi + movq $PMD_PAGE_SIZE, %rcx + rep movsb + + movq %rax, %rsi + movq %r11, %rdi + movq $PMD_PAGE_SIZE, %rcx + rep movsb + + addq $PMD_PAGE_SIZE, %r10 + addq $PMD_PAGE_SIZE, %r11 + subq $PMD_PAGE_SIZE, %r12 + jnz 1b + + /* Restore PAT register */ + push %rdx + movl $MSR_IA32_CR_PAT, %ecx + rdmsr + pop %rdx + wrmsr + + ret +.Lencrypt_stop: +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + .data .align 16 ENTRY(sme_me_mask) @@ -35,3 +339,5 @@ ENTRY(sme_me_mask) sme_me_loss: .byte 0x00 .align 8 +mem_encrypt_enable_option: + .asciz "mem_encrypt=on" -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@kvack.org. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
WARNING: multiple messages have this Message-ID (diff)
From: Tom Lendacky <thomas.lendacky@amd.com> To: linux-arch@vger.kernel.org, linux-efi@vger.kernel.org, kvm@vger.kernel.org, linux-doc@vger.kernel.org, x86@kernel.org, linux-kernel@vger.kernel.org, kasan-dev@googlegroups.com, linux-mm@kvack.org, iommu@lists.linux-foundation.org Cc: "Radim Krčmář" <rkrcmar@redhat.com>, "Arnd Bergmann" <arnd@arndb.de>, "Jonathan Corbet" <corbet@lwn.net>, "Matt Fleming" <matt@codeblueprint.co.uk>, "Joerg Roedel" <joro@8bytes.org>, "Konrad Rzeszutek Wilk" <konrad.wilk@oracle.com>, "Paolo Bonzini" <pbonzini@redhat.com>, "Ingo Molnar" <mingo@redhat.com>, "Borislav Petkov" <bp@alien8.de>, "H. Peter Anvin" <hpa@zytor.com>, "Andrey Ryabinin" <aryabinin@virtuozzo.com>, "Alexander Potapenko" <glider@google.com>, "Thomas Gleixner" <tglx@linutronix.de>, "Dmitry Vyukov" <dvyukov@google.com> Subject: [RFC PATCH v1 18/18] x86: Add support to turn on Secure Memory Encryption Date: Tue, 26 Apr 2016 17:59:04 -0500 [thread overview] Message-ID: <20160426225904.13567.538.stgit@tlendack-t1.amdoffice.net> (raw) Message-ID: <20160426225904.P6eIxlLWYAuZpf60go3OQ6brLVm4LP9JFKN3bZrzx9I@z> (raw) In-Reply-To: <20160426225553.13567.19459.stgit@tlendack-t1.amdoffice.net> This patch adds the support to check for and enable SME when available on the processor and when the mem_encrypt=on command line option is set. This consists of setting the encryption mask, calculating the number of physical bits of addressing lost and encrypting the kernel "in place." Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com> --- Documentation/kernel-parameters.txt | 3 arch/x86/kernel/asm-offsets.c | 2 arch/x86/kernel/mem_encrypt.S | 306 +++++++++++++++++++++++++++++++++++ 3 files changed, 311 insertions(+) diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 8ba7f82..0a2678a 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -2210,6 +2210,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted. memory contents and reserves bad memory regions that are detected. + mem_encrypt=on [X86_64] Enable memory encryption on processors + that support this feature. + meye.*= [HW] Set MotionEye Camera parameters See Documentation/video4linux/meye.txt. diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 5c04246..a0f76de 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -82,6 +82,8 @@ void common(void) { OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment); OFFSET(BP_pref_address, boot_params, hdr.pref_address); OFFSET(BP_code32_start, boot_params, hdr.code32_start); + OFFSET(BP_cmd_line_ptr, boot_params, hdr.cmd_line_ptr); + OFFSET(BP_ext_cmd_line_ptr, boot_params, ext_cmd_line_ptr); BLANK(); DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); diff --git a/arch/x86/kernel/mem_encrypt.S b/arch/x86/kernel/mem_encrypt.S index f2e0536..4d3326d 100644 --- a/arch/x86/kernel/mem_encrypt.S +++ b/arch/x86/kernel/mem_encrypt.S @@ -12,13 +12,236 @@ #include <linux/linkage.h> +#include <asm/processor-flags.h> +#include <asm/pgtable.h> +#include <asm/page.h> +#include <asm/msr.h> +#include <asm/asm-offsets.h> + .text .code64 ENTRY(sme_enable) +#ifdef CONFIG_AMD_MEM_ENCRYPT + /* Check for AMD processor */ + xorl %eax, %eax + cpuid + cmpl $0x68747541, %ebx # AuthenticAMD + jne .Lno_mem_encrypt + cmpl $0x69746e65, %edx + jne .Lno_mem_encrypt + cmpl $0x444d4163, %ecx + jne .Lno_mem_encrypt + + /* Check for memory encryption leaf */ + movl $0x80000000, %eax + cpuid + cmpl $0x8000001f, %eax + jb .Lno_mem_encrypt + + /* + * Check for memory encryption feature: + * CPUID Fn8000_001F[EAX] - Bit 0 + */ + movl $0x8000001f, %eax + cpuid + bt $0, %eax + jnc .Lno_mem_encrypt + + /* Check for the mem_encrypt=on command line option */ + push %rsi /* Save RSI (real_mode_data) */ + movl BP_ext_cmd_line_ptr(%rsi), %ecx + shlq $32, %rcx + movl BP_cmd_line_ptr(%rsi), %edi + addq %rcx, %rdi + leaq mem_encrypt_enable_option(%rip), %rsi + call cmdline_find_option_bool + pop %rsi /* Restore RSI (real_mode_data) */ + testl %eax, %eax + jz .Lno_mem_encrypt + + /* + * Get memory encryption information: + * CPUID Fn8000_001F[EBX] - Bits 5:0 + * Pagetable bit position used to indicate encryption + */ + movl %ebx, %ecx + andl $0x3f, %ecx + jz .Lno_mem_encrypt + bts %ecx, sme_me_mask(%rip) + shrl $6, %ebx + + /* + * Get memory encryption information: + * CPUID Fn8000_001F[EBX] - Bits 11:6 + * Reduction in physical address space (in bits) when enabled + */ + movl %ebx, %ecx + andl $0x3f, %ecx + movb %cl, sme_me_loss(%rip) + + /* + * Enable memory encryption through the SYSCFG MSR + */ + movl $MSR_K8_SYSCFG, %ecx + rdmsr + bt $MSR_K8_SYSCFG_MEM_ENCRYPT_BIT, %eax + jc .Lmem_encrypt_exit + bts $MSR_K8_SYSCFG_MEM_ENCRYPT_BIT, %eax + wrmsr + jmp .Lmem_encrypt_exit + +.Lno_mem_encrypt: + /* Did not get enabled, clear settings */ + movq $0, sme_me_mask(%rip) + movb $0, sme_me_loss(%rip) + +.Lmem_encrypt_exit: +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + ret ENDPROC(sme_enable) ENTRY(sme_encrypt_kernel) +#ifdef CONFIG_AMD_MEM_ENCRYPT + cmpq $0, sme_me_mask(%rip) + jz .Lencrypt_exit + + /* + * Encrypt the kernel. + * Pagetables for performing kernel encryption: + * 0x0000000000 - 0x00FFFFFFFF will map just the memory occupied by + * the kernel as encrypted memory + * 0x8000000000 - 0x80FFFFFFFF will map all memory as write-protected, + * non-encrypted + * + * The use of write-protected memory will prevent any of the + * non-encrypted memory from being cached. + * + * 0x00... and 0x80... represent the first and second PGD entries. + * + * This collection of entries will be created in an area outside + * of the area that is being encrypted (outside the kernel) and + * requires 11 4K pages: + * 1 - PGD + * 2 - PUDs (1 for each mapping) + * 8 - PMDs (4 for each mapping) + */ + leaq _end(%rip), %rdi + addq $~PMD_PAGE_MASK, %rdi + andq $PMD_PAGE_MASK, %rdi /* RDI points to the new PGD */ + + /* Clear the pagetable memory */ + movq %rdi, %rbx /* Save pointer to PGD */ + movl $(4096 * 11), %ecx + xorl %eax, %eax + rep stosb + movq %rbx, %rdi /* Restore pointer to PGD */ + + /* Set up PGD entries for the two mappings */ + leaq (0x1000 + 0x03)(%rdi), %rbx /* PUD for encrypted kernel */ + movq %rbx, (%rdi) + leaq (0x2000 + 0x03)(%rdi), %rbx /* PUD for unencrypted kernel */ + movq %rbx, 8(%rdi) + + /* Set up PUD entries (4 per mapping) for the two mappings */ + leaq (0x3000 + 0x03)(%rdi), %rbx /* PMD for encrypted kernel */ + leaq (0x7000 + 0x03)(%rdi), %rdx /* PMD for unencrypted kernel */ + xorq %rcx, %rcx +1: + /* Populate the PUD entries in each mapping */ + movq %rbx, 0x1000(%rdi, %rcx, 8) + movq %rdx, 0x2000(%rdi, %rcx, 8) + addq $0x1000, %rbx + addq $0x1000, %rdx + incq %rcx + cmpq $4, %rcx + jb 1b + + /* + * Set up PMD entries (4GB worth) for the two mappings. + * For the encrypted kernel mapping, when R11 is above RDX + * and below RDI then we know we are in the kernel and we + * set the encryption mask for that PMD entry. + * + * The use of _PAGE_PAT and _PAGE_PWT will provide for the + * write-protected mapping. + */ + movq sme_me_mask(%rip), %r10 + movq $__PAGE_KERNEL_LARGE_EXEC, %r11 + andq $~_PAGE_GLOBAL, %r11 + movq %r11, %r12 + andq $~_PAGE_CACHE_MASK, %r12 + orq $(_PAGE_PAT | _PAGE_PWT), %r12 /* PA5 index */ + xorq %rcx, %rcx + leaq _text(%rip), %rdx /* RDX points to start of kernel */ +1: + /* Populate the PMD entries in each mapping */ + movq %r11, 0x3000(%rdi, %rcx, 8) + movq %r12, 0x7000(%rdi, %rcx, 8) + + /* + * Check if we are in the kernel range, and if so, set the + * memory encryption mask. + */ + cmpq %r11, %rdx + jae 2f + cmpq %r11, %rdi + jbe 2f + orq %r10, 0x3000(%rdi, %rcx, 8) +2: + addq $PMD_SIZE, %r11 + addq $PMD_SIZE, %r12 + incq %rcx + cmpq $2048, %rcx + jb 1b + + /* + * Set up a one page stack in the non-encrypted memory area. + * Set RAX to point to the next page in memory after all the + * page tables. The stack grows from the bottom so point to + * the end of the page. + */ + leaq (4096 * 11)(%rdi), %rax + addq $PAGE_SIZE, %rax + movq %rsp, %rbp + movq %rax, %rsp + push %rbp /* Save original stack pointer */ + + push %rsi /* Save RSI (real mode data) */ + + /* + * Copy encryption routine into safe memory + * - RAX points to the page after all the page tables and stack + * where the routine will copied + * - RDI points to the PGD table + * - Setup registers for call + * and then call it + */ + movq %rdi, %rbx + + leaq .Lencrypt_start(%rip), %rsi + movq %rax, %rdi + movq $(.Lencrypt_stop - .Lencrypt_start), %rcx + rep movsb + + leaq _text(%rip), %rsi /* Kernel start */ + movq %rbx, %rcx /* New PGD start */ + subq %rsi, %rcx /* Size of area to encrypt */ + + movq %rsi, %rdi /* Encrypted kernel space start */ + movq $1, %rsi + shlq $PGDIR_SHIFT, %rsi + addq %rdi, %rsi /* Non-encrypted kernel start */ + + /* Call the encryption routine */ + call *%rax + + pop %rsi /* Restore RSI (real mode data ) */ + + pop %rsp /* Restore original stack pointer */ +.Lencrypt_exit: +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + ret ENDPROC(sme_encrypt_kernel) @@ -28,6 +251,87 @@ ENTRY(sme_get_me_loss) ret ENDPROC(sme_get_me_loss) +#ifdef CONFIG_AMD_MEM_ENCRYPT +/* + * Routine used to encrypt kernel. + * This routine must be run outside of the kernel proper since + * the kernel will be encrypted during the process. So this + * routine is defined here and then copied to an area outside + * of the kernel where it will remain and run un-encrypted + * during execution. + * + * On entry the registers must be: + * - RAX points to this routine + * - RBX points to new PGD to use + * - RCX contains the kernel length + * - RSI points to the non-encrypted kernel space + * - RDI points to the encrypted kernel space + * + * The kernel will be encrypted by copying from the non-encrypted + * kernel space to a temporary buffer and then copying from the + * temporary buffer back to the encrypted kernel space. The physical + * addresses of the two kernel space mappings are the same which + * results in the kernel being encrypted "in place". + */ +.Lencrypt_start: + /* Enable the new page tables */ + mov %rbx, %cr3 + + /* Flush any global TLBs */ + mov %cr4, %rbx + andq $~X86_CR4_PGE, %rbx + mov %rbx, %cr4 + orq $X86_CR4_PGE, %rbx + mov %rbx, %cr4 + + /* Set the PAT register PA5 entry to write-protect */ + push %rax + push %rcx + movl $MSR_IA32_CR_PAT, %ecx + rdmsr + push %rdx /* Save original PAT value */ + andl $0xffff00ff, %edx /* Clear PA5 */ + orl $0x00000500, %edx /* Set PA5 to WP */ + wrmsr + pop %rdx /* RDX contains original PAT value */ + pop %rcx + pop %rax + + movq %rsi, %r10 /* Save source address */ + movq %rdi, %r11 /* Save destination address */ + movq %rcx, %r12 /* Save length */ + addq $PAGE_SIZE, %rax /* RAX now points to temp copy page */ + + wbinvd /* Invalidate any cache entries */ + + /* Copy/encrypt 2MB at a time */ +1: + movq %r10, %rsi + movq %rax, %rdi + movq $PMD_PAGE_SIZE, %rcx + rep movsb + + movq %rax, %rsi + movq %r11, %rdi + movq $PMD_PAGE_SIZE, %rcx + rep movsb + + addq $PMD_PAGE_SIZE, %r10 + addq $PMD_PAGE_SIZE, %r11 + subq $PMD_PAGE_SIZE, %r12 + jnz 1b + + /* Restore PAT register */ + push %rdx + movl $MSR_IA32_CR_PAT, %ecx + rdmsr + pop %rdx + wrmsr + + ret +.Lencrypt_stop: +#endif /* CONFIG_AMD_MEM_ENCRYPT */ + .data .align 16 ENTRY(sme_me_mask) @@ -35,3 +339,5 @@ ENTRY(sme_me_mask) sme_me_loss: .byte 0x00 .align 8 +mem_encrypt_enable_option: + .asciz "mem_encrypt=on"
next prev parent reply other threads:[~2016-04-26 22:59 UTC|newest] Thread overview: 147+ messages / expand[flat|nested] mbox.gz Atom feed top 2016-04-26 22:55 [RFC PATCH v1 00/18] x86: Secure Memory Encryption (AMD) Tom Lendacky 2016-03-22 13:00 ` Pavel Machek 2016-03-22 13:00 ` Pavel Machek [not found] ` <20160322130058.GA16528-5NIqAleC692hcjWhqY66xCZi+YwRKgec@public.gmane.org> 2016-04-27 14:05 ` Borislav Petkov 2016-04-27 14:05 ` Borislav Petkov 2016-04-27 14:30 ` Pavel Machek 2016-04-27 14:30 ` Pavel Machek 2016-04-27 14:39 ` Borislav Petkov 2016-04-27 14:39 ` Borislav Petkov [not found] ` <20160427143951.GH21011-fF5Pk5pvG8Y@public.gmane.org> 2016-04-27 14:58 ` Pavel Machek 2016-04-27 14:58 ` Pavel Machek 2016-04-27 15:47 ` Pavel Machek 2016-04-27 15:47 ` Pavel Machek 2016-04-27 14:21 ` Tom Lendacky 2016-04-27 14:21 ` Tom Lendacky 2016-04-26 22:55 ` Tom Lendacky 2016-04-26 22:56 ` [RFC PATCH v1 01/18] x86: Set the write-protect cache mode for AMD processors Tom Lendacky 2016-04-26 22:56 ` Tom Lendacky [not found] ` <20160426225604.13567.55443.stgit-qCXWGYdRb2BnqfbPTmsdiZQ+2ll4COg0XqFh9Ls21Oc@public.gmane.org> 2016-04-27 14:33 ` Andy Lutomirski 2016-04-27 14:33 ` Andy Lutomirski 2016-04-27 14:44 ` Tom Lendacky 2016-04-27 14:44 ` Tom Lendacky 2016-04-27 14:47 ` Andy Lutomirski 2016-04-27 14:47 ` Andy Lutomirski [not found] ` <CALCETrV+JzPZjrrqkhWSVfvKQt62Aq8NSW=ZvfdiAi8XKoLi8A-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org> 2016-04-27 15:05 ` Tom Lendacky 2016-04-27 15:05 ` Tom Lendacky 2016-04-27 15:12 ` Andy Lutomirski 2016-04-27 15:12 ` Andy Lutomirski 2016-04-27 15:31 ` Borislav Petkov 2016-04-27 15:31 ` Borislav Petkov 2016-04-27 15:34 ` Andy Lutomirski 2016-04-27 15:34 ` Andy Lutomirski 2016-04-26 22:56 ` [RFC PATCH v1 02/18] x86: Secure Memory Encryption (SME) build enablement Tom Lendacky 2016-03-22 13:01 ` Pavel Machek 2016-03-22 13:01 ` Pavel Machek 2016-04-27 15:17 ` Tom Lendacky 2016-04-27 15:17 ` Tom Lendacky 2016-04-27 15:30 ` Pavel Machek 2016-04-27 15:30 ` Pavel Machek 2016-04-27 15:41 ` Borislav Petkov 2016-04-27 15:41 ` Borislav Petkov 2016-04-27 16:41 ` Pavel Machek 2016-04-27 17:07 ` Robin Murphy 2016-04-27 17:07 ` Robin Murphy 2016-04-27 17:12 ` Borislav Petkov 2016-04-26 22:56 ` Tom Lendacky 2016-04-26 22:56 ` [RFC PATCH v1 03/18] x86: Secure Memory Encryption (SME) support Tom Lendacky 2016-03-22 13:03 ` Pavel Machek 2016-03-22 13:03 ` Pavel Machek 2016-04-27 16:20 ` Tom Lendacky 2016-04-27 16:20 ` Tom Lendacky 2016-04-26 22:56 ` Tom Lendacky 2016-04-26 22:56 ` [RFC PATCH v1 04/18] x86: Add the Secure Memory Encryption cpu feature Tom Lendacky 2016-04-26 22:56 ` Tom Lendacky 2016-04-26 22:56 ` [RFC PATCH v1 05/18] x86: Handle reduction in physical address size with SME Tom Lendacky 2016-04-26 22:56 ` Tom Lendacky 2016-04-26 22:56 ` [RFC PATCH v1 06/18] x86: Provide general kernel support for memory encryption Tom Lendacky 2016-04-26 22:56 ` Tom Lendacky 2016-04-26 22:57 ` [RFC PATCH v1 07/18] x86: Extend the early_memmap support with additional attrs Tom Lendacky 2016-04-26 22:57 ` Tom Lendacky 2016-04-26 22:57 ` [RFC PATCH v1 08/18] x86: Add support for early encryption/decryption of memory Tom Lendacky 2016-04-26 22:57 ` Tom Lendacky 2016-04-26 22:57 ` [RFC PATCH v1 09/18] x86: Insure that memory areas are encrypted when possible Tom Lendacky 2016-04-26 22:57 ` Tom Lendacky 2016-04-26 22:57 ` [RFC PATCH v1 10/18] x86/efi: Access EFI related tables in the clear Tom Lendacky 2016-04-26 22:57 ` Tom Lendacky [not found] ` <20160426225740.13567.85438.stgit-qCXWGYdRb2BnqfbPTmsdiZQ+2ll4COg0XqFh9Ls21Oc@public.gmane.org> 2016-05-10 13:43 ` Matt Fleming 2016-05-10 13:43 ` Matt Fleming [not found] ` <20160510134358.GR2839-mF/unelCI9GS6iBeEJttW/XRex20P6io@public.gmane.org> 2016-05-10 13:57 ` Borislav Petkov 2016-05-10 13:57 ` Borislav Petkov 2016-05-12 18:20 ` Tom Lendacky 2016-05-12 18:20 ` Tom Lendacky 2016-05-24 14:54 ` Tom Lendacky 2016-05-24 14:54 ` Tom Lendacky 2016-05-25 16:09 ` Daniel Kiper 2016-05-25 16:09 ` Daniel Kiper 2016-05-25 19:30 ` Matt Fleming 2016-05-25 19:30 ` Matt Fleming 2016-05-26 13:45 ` Tom Lendacky 2016-05-26 13:45 ` Tom Lendacky 2016-06-08 10:07 ` Matt Fleming 2016-06-08 10:07 ` Matt Fleming 2016-06-09 16:16 ` Tom Lendacky 2016-06-09 16:16 ` Tom Lendacky 2016-06-13 12:03 ` Matt Fleming 2016-06-13 12:03 ` Matt Fleming 2016-06-13 12:34 ` Matt Fleming 2016-06-13 12:34 ` Matt Fleming 2016-06-13 15:16 ` Tom Lendacky 2016-06-13 15:16 ` Tom Lendacky 2016-06-08 11:18 ` Matt Fleming 2016-06-08 11:18 ` Matt Fleming 2016-06-09 18:33 ` Tom Lendacky 2016-06-09 18:33 ` Tom Lendacky 2016-06-13 13:51 ` Matt Fleming 2016-06-13 13:51 ` Matt Fleming [not found] ` <20160613135110.GC2658-mF/unelCI9GS6iBeEJttW/XRex20P6io@public.gmane.org> 2016-06-15 13:17 ` Tom Lendacky 2016-06-15 13:17 ` Tom Lendacky [not found] ` <57615561.4090502-5C7GfCeVMHo@public.gmane.org> 2016-06-16 14:38 ` Tom Lendacky 2016-06-16 14:38 ` Tom Lendacky 2016-06-17 15:51 ` Matt Fleming 2016-06-17 15:51 ` Matt Fleming 2016-04-26 22:57 ` [RFC PATCH v1 11/18] x86: Decrypt trampoline area if memory encryption is active Tom Lendacky 2016-04-26 22:57 ` Tom Lendacky 2016-04-26 22:58 ` [RFC PATCH v1 12/18] x86: Access device tree in the clear Tom Lendacky 2016-04-26 22:58 ` Tom Lendacky 2016-04-26 22:58 ` [RFC PATCH v1 13/18] x86: DMA support for memory encryption Tom Lendacky 2016-04-26 22:58 ` Tom Lendacky 2016-04-29 7:17 ` Konrad Rzeszutek Wilk 2016-04-29 15:12 ` Tom Lendacky 2016-04-29 15:12 ` Tom Lendacky 2016-04-29 16:27 ` Konrad Rzeszutek Wilk [not found] ` <20160429162757.GA1191-he5eyhs8q0BAdwtm4QZOy9BPR1lH4CV8@public.gmane.org> 2016-04-29 23:49 ` Tom Lendacky 2016-04-29 23:49 ` Tom Lendacky 2016-04-26 22:58 ` [RFC PATCH v1 14/18] iommu/amd: AMD IOMMU " Tom Lendacky 2016-04-26 22:58 ` Tom Lendacky 2016-04-26 22:58 ` [RFC PATCH v1 15/18] x86: Enable memory encryption on the APs Tom Lendacky 2016-04-26 22:58 ` Tom Lendacky 2016-05-01 22:10 ` Huang, Kai [not found] ` <f37dd7de-23ad-f70f-c32d-a32f116215ce-VuQAYsv1563Yd54FQh9/CA@public.gmane.org> 2016-05-03 15:59 ` Tom Lendacky 2016-05-03 15:59 ` Tom Lendacky 2016-04-26 22:58 ` [RFC PATCH v1 16/18] x86: Do not specify encrypted memory for VGA mapping Tom Lendacky 2016-04-26 22:58 ` Tom Lendacky 2016-04-26 22:58 ` [RFC PATCH v1 17/18] x86/kvm: Enable Secure Memory Encryption of nested page tables Tom Lendacky 2016-04-26 22:58 ` Tom Lendacky 2016-04-26 22:59 ` Tom Lendacky [this message] [not found] ` <20160426225904.13567.538.stgit-qCXWGYdRb2BnqfbPTmsdiZQ+2ll4COg0XqFh9Ls21Oc@public.gmane.org> 2016-03-22 13:13 ` [RFC PATCH v1 18/18] x86: Add support to turn on Secure Memory Encryption Pavel Machek 2016-03-22 13:13 ` Pavel Machek 2016-04-26 22:59 ` Tom Lendacky [not found] ` <20160426225553.13567.19459.stgit-qCXWGYdRb2BnqfbPTmsdiZQ+2ll4COg0XqFh9Ls21Oc@public.gmane.org> 2016-04-27 14:39 ` [RFC PATCH v1 00/18] x86: Secure Memory Encryption (AMD) Andy Lutomirski 2016-04-27 14:39 ` Andy Lutomirski 2016-04-27 20:10 ` Tom Lendacky 2016-04-27 20:10 ` Tom Lendacky 2016-05-02 18:31 ` Andy Lutomirski 2016-05-02 18:31 ` Andy Lutomirski 2016-05-09 15:13 ` Paolo Bonzini 2016-05-09 15:13 ` Paolo Bonzini 2016-05-09 21:08 ` Tom Lendacky 2016-05-09 21:08 ` Tom Lendacky 2016-05-10 11:23 ` Paolo Bonzini 2016-05-10 11:23 ` Paolo Bonzini 2016-05-10 12:04 ` Borislav Petkov 2016-05-10 12:04 ` Borislav Petkov 2016-04-30 6:13 ` Elliott, Robert (Persistent Memory) 2016-04-30 6:13 ` Elliott, Robert (Persistent Memory) [not found] ` <94D0CD8314A33A4D9D801C0FE68B402963918FDA-wwDBVnaDRpYSZAcGdq5asR6epYMZPwEe5NbjCUgZEJk@public.gmane.org> 2016-05-03 15:55 ` Tom Lendacky 2016-05-03 15:55 ` Tom Lendacky
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20160426225904.13567.538.stgit@tlendack-t1.amdoffice.net \ --to=thomas.lendacky@amd.com \ --cc=arnd@arndb.de \ --cc=aryabinin@virtuozzo.com \ --cc=bp@alien8.de \ --cc=corbet@lwn.net \ --cc=dvyukov@google.com \ --cc=glider@google.com \ --cc=hpa@zytor.com \ --cc=iommu@lists.linux-foundation.org \ --cc=joro@8bytes.org \ --cc=kasan-dev@googlegroups.com \ --cc=konrad.wilk@oracle.com \ --cc=kvm@vger.kernel.org \ --cc=linux-arch@vger.kernel.org \ --cc=linux-doc@vger.kernel.org \ --cc=linux-efi@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=matt@codeblueprint.co.uk \ --cc=mingo@redhat.com \ --cc=pbonzini@redhat.com \ --cc=rkrcmar@redhat.com \ --cc=tglx@linutronix.de \ --cc=x86@kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).