[RFC PATCH v2 04/12] migration: Save confidential guest RAM using migration helper

From: Dov Murik <dovmurik@linux.ibm.com>
To: qemu-devel@nongnu.org
Cc: Tom Lendacky <thomas.lendacky@amd.com>,
	Ashish Kalra <ashish.kalra@amd.com>,
	Brijesh Singh <brijesh.singh@amd.com>,
	"Michael S. Tsirkin" <mst@redhat.com>,
	Steve Rutherford <srutherford@google.com>,
	James Bottomley <jejb@linux.ibm.com>,
	Juan Quintela <quintela@redhat.com>,
	"Dr. David Alan Gilbert" <dgilbert@redhat.com>,
	Dov Murik <dovmurik@linux.ibm.com>,
	Hubertus Franke <frankeh@us.ibm.com>,
	Tobin Feldman-Fitzthum <tobin@linux.ibm.com>,
	Paolo Bonzini <pbonzini@redhat.com>
Subject: [RFC PATCH v2 04/12] migration: Save confidential guest RAM using migration helper
Date: Mon, 23 Aug 2021 10:16:28 -0400	[thread overview]
Message-ID: <20210823141636.65975-5-dovmurik@linux.ibm.com> (raw)
In-Reply-To: <20210823141636.65975-1-dovmurik@linux.ibm.com>

When saving RAM pages of a confidential guest, check whether a page is
encrypted.  If it is, ask the in-guest migration helper to encrypt the
page for transmission.

This patch forces the use of in-guest migration handler instead of the
PSP-based SEV migration; this is just a temporary example.  TODO
introduce migration parameter for choosing this migration mode.

Signed-off-by: Dov Murik <dovmurik@linux.ibm.com>
---
 include/sysemu/sev.h |   1 +
 migration/ram.c      | 109 +++++++++++++++++++++++++++++++++++++++----
 2 files changed, 101 insertions(+), 9 deletions(-)

diff --git a/include/sysemu/sev.h b/include/sysemu/sev.h
index d04890113c..ea52d2f41f 100644
--- a/include/sysemu/sev.h
+++ b/include/sysemu/sev.h
@@ -19,6 +19,7 @@
 
 #define RAM_SAVE_ENCRYPTED_PAGE           0x1
 #define RAM_SAVE_SHARED_REGIONS_LIST      0x2
+#define RAM_SAVE_GUEST_MH_ENCRYPTED_PAGE  0x4
 
 bool sev_enabled(void);
 int sev_kvm_init(ConfidentialGuestSupport *cgs, Error **errp);
diff --git a/migration/ram.c b/migration/ram.c
index 4eca90cceb..a1f89445d4 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -51,12 +51,14 @@
 #include "migration/colo.h"
 #include "block.h"
 #include "sysemu/cpu-throttle.h"
+#include "sysemu/kvm.h"
 #include "savevm.h"
 #include "qemu/iov.h"
 #include "multifd.h"
 #include "sysemu/runstate.h"
 #include "hw/boards.h"
 #include "exec/confidential-guest-support.h"
+#include "confidential-ram.h"
 
 /* Defines RAM_SAVE_ENCRYPTED_PAGE and RAM_SAVE_SHARED_REGION_LIST */
 #include "sysemu/sev.h"
@@ -97,6 +99,13 @@ bool memcrypt_enabled(void)
     return ms->cgs->ready;
 }
 
+static inline bool confidential_guest(void)
+{
+    MachineState *ms = MACHINE(qdev_get_machine());
+
+    return ms->cgs;
+}
+
 XBZRLECacheStats xbzrle_counters;
 
 /* struct contains XBZRLE cache and a static page
@@ -2091,6 +2100,49 @@ static bool encrypted_test_list(RAMState *rs, RAMBlock *block,
     return ops->is_gfn_in_unshared_region(gfn);
 }
 
+/**
+ * ram_save_mh_encrypted_page - use the guest migration handler to encrypt
+ * a page and send it to the stream.
+ *
+ * Return the number of pages written (=1).
+ */
+static int ram_save_mh_encrypted_page(RAMState *rs, PageSearchStatus *pss,
+                                      bool last_stage)
+{
+    int ret;
+    uint8_t *p;
+    RAMBlock *block = pss->block;
+    ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
+    ram_addr_t gpa;
+    uint64_t bytes_sent;
+
+    p = block->host + offset;
+
+    /* Find the GPA of the page */
+    if (!kvm_physical_memory_addr_from_host(kvm_state, p, &gpa)) {
+        error_report("%s failed to get gpa for offset %" PRIu64 " block %s",
+                     __func__, offset, memory_region_name(block->mr));
+        return -1;
+    }
+
+    ram_counters.transferred +=
+        save_page_header(rs, rs->f, block,
+                         offset | RAM_SAVE_FLAG_ENCRYPTED_DATA);
+
+    qemu_put_be32(rs->f, RAM_SAVE_GUEST_MH_ENCRYPTED_PAGE);
+    ram_counters.transferred += sizeof(uint32_t);
+
+    ret = cgs_mh_save_encrypted_page(rs->f, gpa, TARGET_PAGE_SIZE, &bytes_sent);
+    if (ret) {
+        return -1;
+    }
+
+    ram_counters.transferred += bytes_sent;
+    ram_counters.normal++;
+
+    return 1;
+}
+
 /**
  * ram_save_target_page: save one target page
  *
@@ -2111,17 +2163,48 @@ static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss,
         return res;
     }
 
-    /*
-     * If memory encryption is enabled then use memory encryption APIs
-     * to write the outgoing buffer to the wire. The encryption APIs
-     * will take care of accessing the guest memory and re-encrypt it
-     * for the transport purposes.
-     */
-    if (memcrypt_enabled() &&
-        encrypted_test_list(rs, pss->block, pss->page)) {
-        return ram_save_encrypted_page(rs, pss, last_stage);
+    if (confidential_guest()) {
+        /*
+         * TODO: We'd like to support two migration modes for SEV guests:
+         * PSP-based and guest-assisted.  A possible solution is to add a new
+         * migration parameter ("use_guest_assistance") that will controlwhich
+         * mode should be used.
+         */
+        bool guest_assisted_confidential_migration = true;
+
+        if (guest_assisted_confidential_migration) {
+            /*
+             * If memory encryption is enabled then skip saving the data pages
+             * used by the migration handler.
+             */
+            if (gpa_inside_migration_helper_shared_area(offset)) {
+                return 0;
+            }
+
+            /*
+             * If memory encryption is enabled then use in-guest migration
+             * helper to write the outgoing buffer to the wire. The migration
+             * helper will take care of accessing the guest memory and
+             * re-encrypt it for the transport purposes.
+             */
+            if (encrypted_test_list(rs, pss->block, pss->page)) {
+                return ram_save_mh_encrypted_page(rs, pss, last_stage);
+            }
+      } else {
+            /*
+             * If memory encryption is enabled then use memory encryption APIs
+             * to write the outgoing buffer to the wire. The encryption APIs
+             * will take care of accessing the guest memory and re-encrypt it
+             * for the transport purposes.
+             */
+            if (memcrypt_enabled() &&
+                encrypted_test_list(rs, pss->block, pss->page)) {
+                return ram_save_encrypted_page(rs, pss, last_stage);
+            }
+      }
     }
 
+
     if (save_compress_page(rs, block, offset)) {
         return 1;
     }
@@ -2959,6 +3042,10 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
         return -1;
     }
 
+    if (confidential_guest()) {
+        cgs_mh_init();
+    }
+
     /* migration has already setup the bitmap, reuse it. */
     if (!migration_in_colo_state()) {
         if (ram_init_all(rsp) != 0) {
@@ -3167,6 +3254,10 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
         }
     }
 
+    if (confidential_guest()) {
+        cgs_mh_cleanup();
+    }
+
     if (ret >= 0) {
         multifd_send_sync_main(rs->f);
         qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
-- 
2.20.1


