[PATCH v1 0/4] Share common features between AMD SEV / TDX guest

[PATCH v1 0/4] Share common features between AMD SEV / TDX guest

[Kuppuswamy Sathyanarayanan]

Hi All,

Intel's Trust Domain Extensions (TDX) protect guest VMs from malicious
hosts and some physical attacks. TDX has a lot of similarities to AMD SEV.
Features like encryption/decryption and string I/O unroll support can
be shared between these two technologies.

This patch set adds infrastructure changes required to share the code
between AMD SEV and TDX.

Kirill A. Shutemov (1):
  x86: Move common memory encryption code to mem_encrypt.c

Kuppuswamy Sathyanarayanan (3):
  x86/sev: Remove sev_enable_key usage in outs##bwl()/ins##bwl()
  x86/sev: Use CC_ATTR attribute to generalize string I/O unroll
  x86/sev: Rename mem_encrypt.c to mem_encrypt_amd.c

 arch/x86/Kconfig              |  10 +-
 arch/x86/include/asm/io.h     |  20 +-
 arch/x86/kernel/cc_platform.c |   4 +
 arch/x86/mm/Makefile          |   7 +-
 arch/x86/mm/mem_encrypt.c     | 443 +---------------------------------
 arch/x86/mm/mem_encrypt_amd.c | 438 +++++++++++++++++++++++++++++++++
 include/linux/cc_platform.h   |  11 +
 7 files changed, 474 insertions(+), 459 deletions(-)
 create mode 100644 arch/x86/mm/mem_encrypt_amd.c

-- 
2.25.1

[PATCH v1 1/4] x86/sev: Remove sev_enable_key usage in outs##bwl()/ins##bwl()

[Kuppuswamy Sathyanarayanan]

String I/O instructions (INS/OUTS) can be used to move blocks of
data between I/O ports and memory space. But emulation of these
instructions is not supported in AMD SEV platform. Since these
instructions are obsolete, hypervisors rarely emulate them. So to
support the legacy usage, INS/OUTS are unrolled using IN/OUT
instructions.

Currently, this is implemented by adding a SEV specific static
key check in outs##bwl()/ins##bwl() macros. Since TDX VM guests
also need similar support, the implementation needs to be made
generic using the cc_platform_has() call.

In preparation for adding cc_platform_has() based support, as a
first step remove the sev_enable_key usage and replace it with
direct reference to "sev_status".

Since this patch replaces the static key usage, it might lead to
some performance gap.

Suggested-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@linux.intel.com>
---
 arch/x86/include/asm/io.h | 15 ++++++++-------
 arch/x86/mm/mem_encrypt.c | 11 +----------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 5c6a4af0b911..69093a610630 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -257,17 +257,18 @@ static inline void slow_down_io(void)
 #endif
 
 #ifdef CONFIG_AMD_MEM_ENCRYPT
-#include <linux/jump_label.h>
 
-extern struct static_key_false sev_enable_key;
-static inline bool sev_key_active(void)
+extern u64 sev_status;
+
+static inline bool is_sev_enabled(void)
 {
-	return static_branch_unlikely(&sev_enable_key);
+	return ((sev_status & MSR_AMD64_SEV_ENABLED) &&
+		!(sev_status & MSR_AMD64_SEV_ES_ENABLED));
 }
 
 #else /* !CONFIG_AMD_MEM_ENCRYPT */
 
-static inline bool sev_key_active(void) { return false; }
+static inline bool is_sev_enabled(void) { return false; }
 
 #endif /* CONFIG_AMD_MEM_ENCRYPT */
 
@@ -301,7 +302,7 @@ static inline unsigned type in##bwl##_p(int port)			\
 									\
 static inline void outs##bwl(int port, const void *addr, unsigned long count) \
 {									\
-	if (sev_key_active()) {						\
+	if (is_sev_enabled()) {						\
 		unsigned type *value = (unsigned type *)addr;		\
 		while (count) {						\
 			out##bwl(*value, port);				\
@@ -317,7 +318,7 @@ static inline void outs##bwl(int port, const void *addr, unsigned long count) \
 									\
 static inline void ins##bwl(int port, void *addr, unsigned long count)	\
 {									\
-	if (sev_key_active()) {						\
+	if (is_sev_enabled()) {						\
 		unsigned type *value = (unsigned type *)addr;		\
 		while (count) {						\
 			*value = in##bwl(port);				\
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index 35487305d8af..49e5dfc23785 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -43,8 +43,7 @@ u64 sme_me_mask __section(".data") = 0;
 u64 sev_status __section(".data") = 0;
 u64 sev_check_data __section(".data") = 0;
 EXPORT_SYMBOL(sme_me_mask);
-DEFINE_STATIC_KEY_FALSE(sev_enable_key);
-EXPORT_SYMBOL_GPL(sev_enable_key);
+EXPORT_SYMBOL_GPL(sev_status);
 
 /* Buffer used for early in-place encryption by BSP, no locking needed */
 static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
@@ -499,14 +498,6 @@ void __init mem_encrypt_init(void)
 	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
 	swiotlb_update_mem_attributes();
 
-	/*
-	 * With SEV, we need to unroll the rep string I/O instructions,
-	 * but SEV-ES supports them through the #VC handler.
-	 */
-	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) &&
-	    !cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
-		static_branch_enable(&sev_enable_key);
-
 	print_mem_encrypt_feature_info();
 }
 
-- 
2.25.1

[PATCH v1 2/4] x86/sev: Use CC_ATTR attribute to generalize string I/O unroll

[Kuppuswamy Sathyanarayanan]

INS/OUTS instructions are obsolete and hence many hypervisors do not
support its emulation. To support existing usage, string I/O operations
are unrolled using IN/OUT instructions.

AMD SEV platform implements this support by adding unroll logic in
ins#bwl()/outs#bwl() macros with SEV specific checks. Since TDX VM
guests will also need similar support, use
CC_ATTR_GUEST_UNROLL_STRING_IO and generic cc_platform_has() API to
implement it.

Suggested-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@linux.intel.com>
---
 arch/x86/include/asm/io.h     | 21 +++------------------
 arch/x86/kernel/cc_platform.c |  4 ++++
 arch/x86/mm/mem_encrypt.c     |  1 -
 include/linux/cc_platform.h   | 11 +++++++++++
 4 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 69093a610630..f6d91ecb8026 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -40,6 +40,7 @@
 
 #include <linux/string.h>
 #include <linux/compiler.h>
+#include <linux/cc_platform.h>
 #include <asm/page.h>
 #include <asm/early_ioremap.h>
 #include <asm/pgtable_types.h>
@@ -256,22 +257,6 @@ static inline void slow_down_io(void)
 
 #endif
 
-#ifdef CONFIG_AMD_MEM_ENCRYPT
-
-extern u64 sev_status;
-
-static inline bool is_sev_enabled(void)
-{
-	return ((sev_status & MSR_AMD64_SEV_ENABLED) &&
-		!(sev_status & MSR_AMD64_SEV_ES_ENABLED));
-}
-
-#else /* !CONFIG_AMD_MEM_ENCRYPT */
-
-static inline bool is_sev_enabled(void) { return false; }
-
-#endif /* CONFIG_AMD_MEM_ENCRYPT */
-
 #define BUILDIO(bwl, bw, type)						\
 static inline void out##bwl(unsigned type value, int port)		\
 {									\
@@ -302,7 +287,7 @@ static inline unsigned type in##bwl##_p(int port)			\
 									\
 static inline void outs##bwl(int port, const void *addr, unsigned long count) \
 {									\
-	if (is_sev_enabled()) {						\
+	if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) {		\
 		unsigned type *value = (unsigned type *)addr;		\
 		while (count) {						\
 			out##bwl(*value, port);				\
@@ -318,7 +303,7 @@ static inline void outs##bwl(int port, const void *addr, unsigned long count) \
 									\
 static inline void ins##bwl(int port, void *addr, unsigned long count)	\
 {									\
-	if (is_sev_enabled()) {						\
+	if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) {		\
 		unsigned type *value = (unsigned type *)addr;		\
 		while (count) {						\
 			*value = in##bwl(port);				\
diff --git a/arch/x86/kernel/cc_platform.c b/arch/x86/kernel/cc_platform.c
index 03bb2f343ddb..cc1ffe710dd2 100644
--- a/arch/x86/kernel/cc_platform.c
+++ b/arch/x86/kernel/cc_platform.c
@@ -50,6 +50,10 @@ static bool amd_cc_platform_has(enum cc_attr attr)
 	case CC_ATTR_GUEST_STATE_ENCRYPT:
 		return sev_status & MSR_AMD64_SEV_ES_ENABLED;
 
+	case CC_ATTR_GUEST_UNROLL_STRING_IO:
+		return (sev_status & MSR_AMD64_SEV_ENABLED) &&
+			!(sev_status & MSR_AMD64_SEV_ES_ENABLED);
+
 	default:
 		return false;
 	}
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index 49e5dfc23785..b520021a7e7b 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -43,7 +43,6 @@ u64 sme_me_mask __section(".data") = 0;
 u64 sev_status __section(".data") = 0;
 u64 sev_check_data __section(".data") = 0;
 EXPORT_SYMBOL(sme_me_mask);
-EXPORT_SYMBOL_GPL(sev_status);
 
 /* Buffer used for early in-place encryption by BSP, no locking needed */
 static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
diff --git a/include/linux/cc_platform.h b/include/linux/cc_platform.h
index a075b70b9a70..f47f0c9edb3b 100644
--- a/include/linux/cc_platform.h
+++ b/include/linux/cc_platform.h
@@ -61,6 +61,17 @@ enum cc_attr {
 	 * Examples include SEV-ES.
 	 */
 	CC_ATTR_GUEST_STATE_ENCRYPT,
+
+	/**
+	 * @CC_ATTR_GUEST_UNROLL_STRING_IO: String I/O is implemented with
+	 *                                  IN/OUT instructions
+	 *
+	 * The platform/OS is running as a guest/virtual machine and uses
+	 * IN/OUT instructions in place of string I/O.
+	 *
+	 * Examples include TDX Guest & SEV.
+	 */
+	CC_ATTR_GUEST_UNROLL_STRING_IO,
 };
 
 #ifdef CONFIG_ARCH_HAS_CC_PLATFORM
-- 
2.25.1

[PATCH v1 3/4] x86/sev: Rename mem_encrypt.c to mem_encrypt_amd.c

[Kuppuswamy Sathyanarayanan]

Both TDX and AMD SEV/SME implement memory encryption features. But
the bulk of the code in mem_encrypt.c is AMD specific. Rename the
file to mem_encrypt_amd.c. A subsequent patch will extract the
parts that can be shared by both TDX and AMD SEV/SME into a generic
file.

No functional changes.

Reviewed-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@linux.intel.com>
---
 arch/x86/mm/Makefile                             | 8 ++++----
 arch/x86/mm/{mem_encrypt.c => mem_encrypt_amd.c} | 0
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename arch/x86/mm/{mem_encrypt.c => mem_encrypt_amd.c} (100%)

diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index 5864219221ca..c9c480641153 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 # Kernel does not boot with instrumentation of tlb.c and mem_encrypt*.c
 KCOV_INSTRUMENT_tlb.o			:= n
-KCOV_INSTRUMENT_mem_encrypt.o		:= n
+KCOV_INSTRUMENT_mem_encrypt_amd.o	:= n
 KCOV_INSTRUMENT_mem_encrypt_identity.o	:= n
 
-KASAN_SANITIZE_mem_encrypt.o		:= n
+KASAN_SANITIZE_mem_encrypt_amd.o	:= n
 KASAN_SANITIZE_mem_encrypt_identity.o	:= n
 
 # Disable KCSAN entirely, because otherwise we get warnings that some functions
@@ -12,7 +12,7 @@ KASAN_SANITIZE_mem_encrypt_identity.o	:= n
 KCSAN_SANITIZE := n
 
 ifdef CONFIG_FUNCTION_TRACER
-CFLAGS_REMOVE_mem_encrypt.o		= -pg
+CFLAGS_REMOVE_mem_encrypt_amd.o		= -pg
 CFLAGS_REMOVE_mem_encrypt_identity.o	= -pg
 endif
 
@@ -52,6 +52,6 @@ obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)	+= pkeys.o
 obj-$(CONFIG_RANDOMIZE_MEMORY)			+= kaslr.o
 obj-$(CONFIG_PAGE_TABLE_ISOLATION)		+= pti.o
 
-obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt.o
+obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_amd.o
 obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_identity.o
 obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_boot.o
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt_amd.c
similarity index 100%
rename from arch/x86/mm/mem_encrypt.c
rename to arch/x86/mm/mem_encrypt_amd.c
-- 
2.25.1

[PATCH v1 4/4] x86: Move common memory encryption code to mem_encrypt.c

[Kuppuswamy Sathyanarayanan]

From: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>

SEV and TDX both protect guest memory from the host access. They both
use guest physical address bits to communicate to the hardware which
pages receive protection or not. SEV and TDX both assume that all I/O
(real devices and virtio) must be performed to pages *without*
protection.

To add this support, AMD SEV code forces force_dma_unencrypted() to
decrypt DMA pages when DMA pages were allocated for I/O. It also uses
swiotlb_update_mem_attributes() to update decryption bits in SWIOTLB
DMA buffers.

Since TDX also uses a similar memory sharing design, all the above
mentioned changes can be reused. So move force_dma_unencrypted(),
SWIOTLB update code and virtio changes out of mem_encrypt_amd.c to
mem_encrypt.c.

Introduce a new config option X86_MEM_ENCRYPT that can be selected
by platforms which uses x86 memory encryption features (needed in both
AMD SEV and Intel TDX guest platforms).

Since the code is moved from mem_encrypt_amdc.c, inherit the same make
flags.

This is preparation for enabling TDX memory encryption support and it
has no functional changes.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reviewed-by: Andi Kleen <ak@linux.intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Signed-off-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@linux.intel.com>
---
 arch/x86/Kconfig              | 10 +++--
 arch/x86/mm/Makefile          |  5 +++
 arch/x86/mm/mem_encrypt.c     | 84 +++++++++++++++++++++++++++++++++++
 arch/x86/mm/mem_encrypt_amd.c | 69 ----------------------------
 4 files changed, 96 insertions(+), 72 deletions(-)
 create mode 100644 arch/x86/mm/mem_encrypt.c

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 95dd1ee01546..793e9b42ace0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1523,16 +1523,20 @@ config X86_CPA_STATISTICS
 	  helps to determine the effectiveness of preserving large and huge
 	  page mappings when mapping protections are changed.
 
+config X86_MEM_ENCRYPT
+	select ARCH_HAS_FORCE_DMA_UNENCRYPTED
+	select DYNAMIC_PHYSICAL_MASK
+	select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS
+	def_bool n
+
 config AMD_MEM_ENCRYPT
 	bool "AMD Secure Memory Encryption (SME) support"
 	depends on X86_64 && CPU_SUP_AMD
 	select DMA_COHERENT_POOL
-	select DYNAMIC_PHYSICAL_MASK
 	select ARCH_USE_MEMREMAP_PROT
-	select ARCH_HAS_FORCE_DMA_UNENCRYPTED
 	select INSTRUCTION_DECODER
-	select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS
 	select ARCH_HAS_CC_PLATFORM
+	select X86_MEM_ENCRYPT
 	help
 	  Say yes to enable support for the encryption of system memory.
 	  This requires an AMD processor that supports Secure Memory
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index c9c480641153..fe3d3061fc11 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0
 # Kernel does not boot with instrumentation of tlb.c and mem_encrypt*.c
 KCOV_INSTRUMENT_tlb.o			:= n
+KCOV_INSTRUMENT_mem_encrypt.o		:= n
 KCOV_INSTRUMENT_mem_encrypt_amd.o	:= n
 KCOV_INSTRUMENT_mem_encrypt_identity.o	:= n
 
+KASAN_SANITIZE_mem_encrypt.o		:= n
 KASAN_SANITIZE_mem_encrypt_amd.o	:= n
 KASAN_SANITIZE_mem_encrypt_identity.o	:= n
 
@@ -12,6 +14,7 @@ KASAN_SANITIZE_mem_encrypt_identity.o	:= n
 KCSAN_SANITIZE := n
 
 ifdef CONFIG_FUNCTION_TRACER
+CFLAGS_REMOVE_mem_encrypt.o		= -pg
 CFLAGS_REMOVE_mem_encrypt_amd.o		= -pg
 CFLAGS_REMOVE_mem_encrypt_identity.o	= -pg
 endif
@@ -52,6 +55,8 @@ obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)	+= pkeys.o
 obj-$(CONFIG_RANDOMIZE_MEMORY)			+= kaslr.o
 obj-$(CONFIG_PAGE_TABLE_ISOLATION)		+= pti.o
 
+obj-$(CONFIG_X86_MEM_ENCRYPT)	+= mem_encrypt.o
 obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_amd.o
+
 obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_identity.o
 obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_boot.o
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
new file mode 100644
index 000000000000..5ae4b3d7d549
--- /dev/null
+++ b/arch/x86/mm/mem_encrypt.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Memory Encryption Support Common Code
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ */
+
+#include <linux/dma-direct.h>
+#include <linux/dma-mapping.h>
+#include <linux/swiotlb.h>
+#include <linux/cc_platform.h>
+#include <linux/mem_encrypt.h>
+#include <linux/virtio_config.h>
+
+/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
+bool force_dma_unencrypted(struct device *dev)
+{
+	/*
+	 * For SEV, all DMA must be to unencrypted addresses.
+	 */
+	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
+		return true;
+
+	/*
+	 * For SME, all DMA must be to unencrypted addresses if the
+	 * device does not support DMA to addresses that include the
+	 * encryption mask.
+	 */
+	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
+		u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
+		u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
+						dev->bus_dma_limit);
+
+		if (dma_dev_mask <= dma_enc_mask)
+			return true;
+	}
+
+	return false;
+}
+
+static void print_amd_mem_encrypt_feature_info(void)
+{
+	pr_info("AMD Memory Encryption Features active:");
+
+	/* Secure Memory Encryption */
+	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
+		/*
+		 * SME is mutually exclusive with any of the SEV
+		 * features below.
+		 */
+		pr_cont(" SME\n");
+		return;
+	}
+
+	/* Secure Encrypted Virtualization */
+	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
+		pr_cont(" SEV");
+
+	/* Encrypted Register State */
+	if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
+		pr_cont(" SEV-ES");
+
+	pr_cont("\n");
+}
+
+/* Architecture __weak replacement functions */
+void __init mem_encrypt_init(void)
+{
+	if (!sme_me_mask)
+		return;
+
+	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
+	swiotlb_update_mem_attributes();
+
+	print_amd_mem_encrypt_feature_info();
+}
+
+int arch_has_restricted_virtio_memory_access(void)
+{
+	return cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT);
+}
+EXPORT_SYMBOL_GPL(arch_has_restricted_virtio_memory_access);
diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
index b520021a7e7b..2b2d018ea345 100644
--- a/arch/x86/mm/mem_encrypt_amd.c
+++ b/arch/x86/mm/mem_encrypt_amd.c
@@ -413,32 +413,6 @@ void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, boo
 	notify_range_enc_status_changed(vaddr, npages, enc);
 }
 
-/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
-bool force_dma_unencrypted(struct device *dev)
-{
-	/*
-	 * For SEV, all DMA must be to unencrypted addresses.
-	 */
-	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
-		return true;
-
-	/*
-	 * For SME, all DMA must be to unencrypted addresses if the
-	 * device does not support DMA to addresses that include the
-	 * encryption mask.
-	 */
-	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
-		u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
-		u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
-						dev->bus_dma_limit);
-
-		if (dma_dev_mask <= dma_enc_mask)
-			return true;
-	}
-
-	return false;
-}
-
 void __init mem_encrypt_free_decrypted_mem(void)
 {
 	unsigned long vaddr, vaddr_end, npages;
@@ -462,46 +436,3 @@ void __init mem_encrypt_free_decrypted_mem(void)
 
 	free_init_pages("unused decrypted", vaddr, vaddr_end);
 }
-
-static void print_mem_encrypt_feature_info(void)
-{
-	pr_info("AMD Memory Encryption Features active:");
-
-	/* Secure Memory Encryption */
-	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
-		/*
-		 * SME is mutually exclusive with any of the SEV
-		 * features below.
-		 */
-		pr_cont(" SME\n");
-		return;
-	}
-
-	/* Secure Encrypted Virtualization */
-	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
-		pr_cont(" SEV");
-
-	/* Encrypted Register State */
-	if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
-		pr_cont(" SEV-ES");
-
-	pr_cont("\n");
-}
-
-/* Architecture __weak replacement functions */
-void __init mem_encrypt_init(void)
-{
-	if (!sme_me_mask)
-		return;
-
-	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
-	swiotlb_update_mem_attributes();
-
-	print_mem_encrypt_feature_info();
-}
-
-int arch_has_restricted_virtio_memory_access(void)
-{
-	return cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT);
-}
-EXPORT_SYMBOL_GPL(arch_has_restricted_virtio_memory_access);
-- 
2.25.1

Re: [PATCH v1 0/4] Share common features between AMD SEV / TDX guest

[Kirill A. Shutemov]

On Mon, Nov 15, 2021 at 04:45:24PM -0800, Kuppuswamy Sathyanarayanan wrote:
> Hi All,
> 
> Intel's Trust Domain Extensions (TDX) protect guest VMs from malicious
> hosts and some physical attacks. TDX has a lot of similarities to AMD SEV.
> Features like encryption/decryption and string I/O unroll support can
> be shared between these two technologies.
> 
> This patch set adds infrastructure changes required to share the code
> between AMD SEV and TDX.

Tom, Joerg, could you folks look at this? Your Ack would be very helpful.

-- 
 Kirill A. Shutemov

Re: [PATCH v1 1/4] x86/sev: Remove sev_enable_key usage in outs##bwl()/ins##bwl()

[Tom Lendacky]

On 11/15/21 6:45 PM, Kuppuswamy Sathyanarayanan wrote:
> String I/O instructions (INS/OUTS) can be used to move blocks of
> data between I/O ports and memory space. But emulation of these
> instructions is not supported in AMD SEV platform. Since these
> instructions are obsolete, hypervisors rarely emulate them. So to
> support the legacy usage, INS/OUTS are unrolled using IN/OUT
> instructions.
> 
> Currently, this is implemented by adding a SEV specific static
> key check in outs##bwl()/ins##bwl() macros. Since TDX VM guests
> also need similar support, the implementation needs to be made
> generic using the cc_platform_has() call.
> 
> In preparation for adding cc_platform_has() based support, as a
> first step remove the sev_enable_key usage and replace it with
> direct reference to "sev_status".

Probably makes sense to just go directly to the cc_platform_has() call 
rather than introduce the usage of sev_status.

So maybe squash patches 1 and 2 into a single patch.

Thanks,
Tom

> 
> Since this patch replaces the static key usage, it might lead to
> some performance gap.
> 
> Suggested-by: Tom Lendacky <thomas.lendacky@amd.com>
> Reviewed-by: Tony Luck <tony.luck@intel.com>
> Signed-off-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@linux.intel.com>
> ---
>   arch/x86/include/asm/io.h | 15 ++++++++-------
>   arch/x86/mm/mem_encrypt.c | 11 +----------
>   2 files changed, 9 insertions(+), 17 deletions(-)
> 
> diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
> index 5c6a4af0b911..69093a610630 100644
> --- a/arch/x86/include/asm/io.h
> +++ b/arch/x86/include/asm/io.h
> @@ -257,17 +257,18 @@ static inline void slow_down_io(void)
>   #endif
>   
>   #ifdef CONFIG_AMD_MEM_ENCRYPT
> -#include <linux/jump_label.h>
>   
> -extern struct static_key_false sev_enable_key;
> -static inline bool sev_key_active(void)
> +extern u64 sev_status;
> +
> +static inline bool is_sev_enabled(void)
>   {
> -	return static_branch_unlikely(&sev_enable_key);
> +	return ((sev_status & MSR_AMD64_SEV_ENABLED) &&
> +		!(sev_status & MSR_AMD64_SEV_ES_ENABLED));
>   }
>   
>   #else /* !CONFIG_AMD_MEM_ENCRYPT */
>   
> -static inline bool sev_key_active(void) { return false; }
> +static inline bool is_sev_enabled(void) { return false; }
>   
>   #endif /* CONFIG_AMD_MEM_ENCRYPT */
>   
> @@ -301,7 +302,7 @@ static inline unsigned type in##bwl##_p(int port)			\
>   									\
>   static inline void outs##bwl(int port, const void *addr, unsigned long count) \
>   {									\
> -	if (sev_key_active()) {						\
> +	if (is_sev_enabled()) {						\
>   		unsigned type *value = (unsigned type *)addr;		\
>   		while (count) {						\
>   			out##bwl(*value, port);				\
> @@ -317,7 +318,7 @@ static inline void outs##bwl(int port, const void *addr, unsigned long count) \
>   									\
>   static inline void ins##bwl(int port, void *addr, unsigned long count)	\
>   {									\
> -	if (sev_key_active()) {						\
> +	if (is_sev_enabled()) {						\
>   		unsigned type *value = (unsigned type *)addr;		\
>   		while (count) {						\
>   			*value = in##bwl(port);				\
> diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
> index 35487305d8af..49e5dfc23785 100644
> --- a/arch/x86/mm/mem_encrypt.c
> +++ b/arch/x86/mm/mem_encrypt.c
> @@ -43,8 +43,7 @@ u64 sme_me_mask __section(".data") = 0;
>   u64 sev_status __section(".data") = 0;
>   u64 sev_check_data __section(".data") = 0;
>   EXPORT_SYMBOL(sme_me_mask);
> -DEFINE_STATIC_KEY_FALSE(sev_enable_key);
> -EXPORT_SYMBOL_GPL(sev_enable_key);
> +EXPORT_SYMBOL_GPL(sev_status);
>   
>   /* Buffer used for early in-place encryption by BSP, no locking needed */
>   static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
> @@ -499,14 +498,6 @@ void __init mem_encrypt_init(void)
>   	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
>   	swiotlb_update_mem_attributes();
>   
> -	/*
> -	 * With SEV, we need to unroll the rep string I/O instructions,
> -	 * but SEV-ES supports them through the #VC handler.
> -	 */
> -	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) &&
> -	    !cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
> -		static_branch_enable(&sev_enable_key);
> -
>   	print_mem_encrypt_feature_info();
>   }
>   
> 

Re: [PATCH v1 4/4] x86: Move common memory encryption code to mem_encrypt.c

[Tom Lendacky]

On 11/15/21 6:45 PM, Kuppuswamy Sathyanarayanan wrote:
> From: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
> 
> SEV and TDX both protect guest memory from the host access. They both
> use guest physical address bits to communicate to the hardware which
> pages receive protection or not. SEV and TDX both assume that all I/O
> (real devices and virtio) must be performed to pages *without*
> protection.
> 
> To add this support, AMD SEV code forces force_dma_unencrypted() to
> decrypt DMA pages when DMA pages were allocated for I/O. It also uses
> swiotlb_update_mem_attributes() to update decryption bits in SWIOTLB
> DMA buffers.
> 
> Since TDX also uses a similar memory sharing design, all the above
> mentioned changes can be reused. So move force_dma_unencrypted(),
> SWIOTLB update code and virtio changes out of mem_encrypt_amd.c to
> mem_encrypt.c.
> 
> Introduce a new config option X86_MEM_ENCRYPT that can be selected
> by platforms which uses x86 memory encryption features (needed in both
> AMD SEV and Intel TDX guest platforms).
> 
> Since the code is moved from mem_encrypt_amdc.c, inherit the same make
> flags.
> 
> This is preparation for enabling TDX memory encryption support and it
> has no functional changes.

It seems strange that some AMD specific function/code is in the common 
file, especially print_amd_mem_encrypt_feature_info(). I'm not sure if it 
would be better to wait and move that and mem_encrypt_init() when you add 
the TDX support, since you'll have to update mem_encrypt_init() to change 
the check anyway. Just my opinion, though.

Thanks,
Tom

> 
> Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
> Reviewed-by: Andi Kleen <ak@linux.intel.com>
> Reviewed-by: Tony Luck <tony.luck@intel.com>
> Signed-off-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@linux.intel.com>
> ---
>   arch/x86/Kconfig              | 10 +++--
>   arch/x86/mm/Makefile          |  5 +++
>   arch/x86/mm/mem_encrypt.c     | 84 +++++++++++++++++++++++++++++++++++
>   arch/x86/mm/mem_encrypt_amd.c | 69 ----------------------------
>   4 files changed, 96 insertions(+), 72 deletions(-)
>   create mode 100644 arch/x86/mm/mem_encrypt.c
> 
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index 95dd1ee01546..793e9b42ace0 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -1523,16 +1523,20 @@ config X86_CPA_STATISTICS
>   	  helps to determine the effectiveness of preserving large and huge
>   	  page mappings when mapping protections are changed.
>   
> +config X86_MEM_ENCRYPT
> +	select ARCH_HAS_FORCE_DMA_UNENCRYPTED
> +	select DYNAMIC_PHYSICAL_MASK
> +	select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS
> +	def_bool n
> +
>   config AMD_MEM_ENCRYPT
>   	bool "AMD Secure Memory Encryption (SME) support"
>   	depends on X86_64 && CPU_SUP_AMD
>   	select DMA_COHERENT_POOL
> -	select DYNAMIC_PHYSICAL_MASK
>   	select ARCH_USE_MEMREMAP_PROT
> -	select ARCH_HAS_FORCE_DMA_UNENCRYPTED
>   	select INSTRUCTION_DECODER
> -	select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS
>   	select ARCH_HAS_CC_PLATFORM
> +	select X86_MEM_ENCRYPT
>   	help
>   	  Say yes to enable support for the encryption of system memory.
>   	  This requires an AMD processor that supports Secure Memory
> diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
> index c9c480641153..fe3d3061fc11 100644
> --- a/arch/x86/mm/Makefile
> +++ b/arch/x86/mm/Makefile
> @@ -1,9 +1,11 @@
>   # SPDX-License-Identifier: GPL-2.0
>   # Kernel does not boot with instrumentation of tlb.c and mem_encrypt*.c
>   KCOV_INSTRUMENT_tlb.o			:= n
> +KCOV_INSTRUMENT_mem_encrypt.o		:= n
>   KCOV_INSTRUMENT_mem_encrypt_amd.o	:= n
>   KCOV_INSTRUMENT_mem_encrypt_identity.o	:= n
>   
> +KASAN_SANITIZE_mem_encrypt.o		:= n
>   KASAN_SANITIZE_mem_encrypt_amd.o	:= n
>   KASAN_SANITIZE_mem_encrypt_identity.o	:= n
>   
> @@ -12,6 +14,7 @@ KASAN_SANITIZE_mem_encrypt_identity.o	:= n
>   KCSAN_SANITIZE := n
>   
>   ifdef CONFIG_FUNCTION_TRACER
> +CFLAGS_REMOVE_mem_encrypt.o		= -pg
>   CFLAGS_REMOVE_mem_encrypt_amd.o		= -pg
>   CFLAGS_REMOVE_mem_encrypt_identity.o	= -pg
>   endif
> @@ -52,6 +55,8 @@ obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS)	+= pkeys.o
>   obj-$(CONFIG_RANDOMIZE_MEMORY)			+= kaslr.o
>   obj-$(CONFIG_PAGE_TABLE_ISOLATION)		+= pti.o
>   
> +obj-$(CONFIG_X86_MEM_ENCRYPT)	+= mem_encrypt.o
>   obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_amd.o
> +
>   obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_identity.o
>   obj-$(CONFIG_AMD_MEM_ENCRYPT)	+= mem_encrypt_boot.o
> diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
> new file mode 100644
> index 000000000000..5ae4b3d7d549
> --- /dev/null
> +++ b/arch/x86/mm/mem_encrypt.c
> @@ -0,0 +1,84 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Memory Encryption Support Common Code
> + *
> + * Copyright (C) 2016 Advanced Micro Devices, Inc.
> + *
> + * Author: Tom Lendacky <thomas.lendacky@amd.com>
> + */
> +
> +#include <linux/dma-direct.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/swiotlb.h>
> +#include <linux/cc_platform.h>
> +#include <linux/mem_encrypt.h>
> +#include <linux/virtio_config.h>
> +
> +/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
> +bool force_dma_unencrypted(struct device *dev)
> +{
> +	/*
> +	 * For SEV, all DMA must be to unencrypted addresses.
> +	 */
> +	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
> +		return true;
> +
> +	/*
> +	 * For SME, all DMA must be to unencrypted addresses if the
> +	 * device does not support DMA to addresses that include the
> +	 * encryption mask.
> +	 */
> +	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
> +		u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
> +		u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
> +						dev->bus_dma_limit);
> +
> +		if (dma_dev_mask <= dma_enc_mask)
> +			return true;
> +	}
> +
> +	return false;
> +}
> +
> +static void print_amd_mem_encrypt_feature_info(void)
> +{
> +	pr_info("AMD Memory Encryption Features active:");
> +
> +	/* Secure Memory Encryption */
> +	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
> +		/*
> +		 * SME is mutually exclusive with any of the SEV
> +		 * features below.
> +		 */
> +		pr_cont(" SME\n");
> +		return;
> +	}
> +
> +	/* Secure Encrypted Virtualization */
> +	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
> +		pr_cont(" SEV");
> +
> +	/* Encrypted Register State */
> +	if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
> +		pr_cont(" SEV-ES");
> +
> +	pr_cont("\n");
> +}
> +
> +/* Architecture __weak replacement functions */
> +void __init mem_encrypt_init(void)
> +{
> +	if (!sme_me_mask)
> +		return;
> +
> +	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
> +	swiotlb_update_mem_attributes();
> +
> +	print_amd_mem_encrypt_feature_info();
> +}
> +
> +int arch_has_restricted_virtio_memory_access(void)
> +{
> +	return cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT);
> +}
> +EXPORT_SYMBOL_GPL(arch_has_restricted_virtio_memory_access);
> diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
> index b520021a7e7b..2b2d018ea345 100644
> --- a/arch/x86/mm/mem_encrypt_amd.c
> +++ b/arch/x86/mm/mem_encrypt_amd.c
> @@ -413,32 +413,6 @@ void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, boo
>   	notify_range_enc_status_changed(vaddr, npages, enc);
>   }
>   
> -/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
> -bool force_dma_unencrypted(struct device *dev)
> -{
> -	/*
> -	 * For SEV, all DMA must be to unencrypted addresses.
> -	 */
> -	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
> -		return true;
> -
> -	/*
> -	 * For SME, all DMA must be to unencrypted addresses if the
> -	 * device does not support DMA to addresses that include the
> -	 * encryption mask.
> -	 */
> -	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
> -		u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
> -		u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
> -						dev->bus_dma_limit);
> -
> -		if (dma_dev_mask <= dma_enc_mask)
> -			return true;
> -	}
> -
> -	return false;
> -}
> -
>   void __init mem_encrypt_free_decrypted_mem(void)
>   {
>   	unsigned long vaddr, vaddr_end, npages;
> @@ -462,46 +436,3 @@ void __init mem_encrypt_free_decrypted_mem(void)
>   
>   	free_init_pages("unused decrypted", vaddr, vaddr_end);
>   }
> -
> -static void print_mem_encrypt_feature_info(void)
> -{
> -	pr_info("AMD Memory Encryption Features active:");
> -
> -	/* Secure Memory Encryption */
> -	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
> -		/*
> -		 * SME is mutually exclusive with any of the SEV
> -		 * features below.
> -		 */
> -		pr_cont(" SME\n");
> -		return;
> -	}
> -
> -	/* Secure Encrypted Virtualization */
> -	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
> -		pr_cont(" SEV");
> -
> -	/* Encrypted Register State */
> -	if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
> -		pr_cont(" SEV-ES");
> -
> -	pr_cont("\n");
> -}
> -
> -/* Architecture __weak replacement functions */
> -void __init mem_encrypt_init(void)
> -{
> -	if (!sme_me_mask)
> -		return;
> -
> -	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
> -	swiotlb_update_mem_attributes();
> -
> -	print_mem_encrypt_feature_info();
> -}
> -
> -int arch_has_restricted_virtio_memory_access(void)
> -{
> -	return cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT);
> -}
> -EXPORT_SYMBOL_GPL(arch_has_restricted_virtio_memory_access);
> 

Re: [PATCH v1 0/4] Share common features between AMD SEV / TDX guest

[Tom Lendacky]

On 12/1/21 10:34 AM, Kirill A. Shutemov wrote:
> On Mon, Nov 15, 2021 at 04:45:24PM -0800, Kuppuswamy Sathyanarayanan wrote:
>> Hi All,
>>
>> Intel's Trust Domain Extensions (TDX) protect guest VMs from malicious
>> hosts and some physical attacks. TDX has a lot of similarities to AMD SEV.
>> Features like encryption/decryption and string I/O unroll support can
>> be shared between these two technologies.
>>
>> This patch set adds infrastructure changes required to share the code
>> between AMD SEV and TDX.
> 
> Tom, Joerg, could you folks look at this? Your Ack would be very helpful.

I had a couple of comments on the patches, but they did build and test 
successfully with both SEV and SEV-ES guests.

Thanks,
Tom

>