From: Will Deacon <will@kernel.org>
To: kvmarm@lists.linux.dev
Cc: Will Deacon <will@kernel.org>,
Sean Christopherson <seanjc@google.com>,
Vincent Donnefort <vdonnefort@google.com>,
Alexandru Elisei <alexandru.elisei@arm.com>,
Catalin Marinas <catalin.marinas@arm.com>,
James Morse <james.morse@arm.com>,
Chao Peng <chao.p.peng@linux.intel.com>,
Quentin Perret <qperret@google.com>,
Suzuki K Poulose <suzuki.poulose@arm.com>,
Mark Rutland <mark.rutland@arm.com>,
Fuad Tabba <tabba@google.com>,
Oliver Upton <oliver.upton@linux.dev>,
Marc Zyngier <maz@kernel.org>,
kernel-team@android.com, kvm@vger.kernel.org,
linux-arm-kernel@lists.infradead.org
Subject: [PATCH v4 19/25] KVM: arm64: Instantiate guest stage-2 page-tables at EL2
Date: Mon, 17 Oct 2022 12:52:03 +0100 [thread overview]
Message-ID: <20221017115209.2099-20-will@kernel.org> (raw)
In-Reply-To: <20221017115209.2099-1-will@kernel.org>
From: Quentin Perret <qperret@google.com>
Extend the initialisation of guest data structures within the pKVM
hypervisor at EL2 so that we instantiate a memory pool and a full
'struct kvm_s2_mmu' structure for each VM, with a stage-2 page-table
entirely independent from the one managed by the host at EL1.
The 'struct kvm_pgtable_mm_ops' used by the page-table code is populated
with a set of callbacks that can manage guest pages in the hypervisor
without any direct intervention from the host, allocating page-table
pages from the provided pool and returning these to the host on VM
teardown. To keep things simple, the stage-2 MMU for the guest is
configured identically to the host stage-2 in the VTCR register and so
the IPA size of the guest must match the PA size of the host.
For now, the new page-table is unused as there is no way for the host
to map anything into it. Yet.
Tested-by: Vincent Donnefort <vdonnefort@google.com>
Signed-off-by: Quentin Perret <qperret@google.com>
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 6 ++
arch/arm64/kvm/hyp/nvhe/mem_protect.c | 125 ++++++++++++++++++++++++-
arch/arm64/kvm/mmu.c | 4 +-
3 files changed, 132 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
index d176399dbfb5..5d456438445c 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
@@ -9,6 +9,9 @@
#include <asm/kvm_pkvm.h>
+#include <nvhe/gfp.h>
+#include <nvhe/spinlock.h>
+
/*
* Holds the relevant data for maintaining the vcpu state completely at hyp.
*/
@@ -36,6 +39,9 @@ struct pkvm_hyp_vm {
/* The guest's stage-2 page-table managed by the hypervisor. */
struct kvm_pgtable pgt;
+ struct kvm_pgtable_mm_ops mm_ops;
+ struct hyp_pool pool;
+ hyp_spinlock_t lock;
/*
* The number of vcpus initialized and ready to run.
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 1c38451050e5..27b16a6b85bb 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -25,6 +25,21 @@ struct host_mmu host_mmu;
static struct hyp_pool host_s2_pool;
+static DEFINE_PER_CPU(struct pkvm_hyp_vm *, __current_vm);
+#define current_vm (*this_cpu_ptr(&__current_vm))
+
+static void guest_lock_component(struct pkvm_hyp_vm *vm)
+{
+ hyp_spin_lock(&vm->lock);
+ current_vm = vm;
+}
+
+static void guest_unlock_component(struct pkvm_hyp_vm *vm)
+{
+ current_vm = NULL;
+ hyp_spin_unlock(&vm->lock);
+}
+
static void host_lock_component(void)
{
hyp_spin_lock(&host_mmu.lock);
@@ -140,18 +155,124 @@ int kvm_host_prepare_stage2(void *pgt_pool_base)
return 0;
}
+static bool guest_stage2_force_pte_cb(u64 addr, u64 end,
+ enum kvm_pgtable_prot prot)
+{
+ return true;
+}
+
+static void *guest_s2_zalloc_pages_exact(size_t size)
+{
+ void *addr = hyp_alloc_pages(¤t_vm->pool, get_order(size));
+
+ WARN_ON(size != (PAGE_SIZE << get_order(size)));
+ hyp_split_page(hyp_virt_to_page(addr));
+
+ return addr;
+}
+
+static void guest_s2_free_pages_exact(void *addr, unsigned long size)
+{
+ u8 order = get_order(size);
+ unsigned int i;
+
+ for (i = 0; i < (1 << order); i++)
+ hyp_put_page(¤t_vm->pool, addr + (i * PAGE_SIZE));
+}
+
+static void *guest_s2_zalloc_page(void *mc)
+{
+ struct hyp_page *p;
+ void *addr;
+
+ addr = hyp_alloc_pages(¤t_vm->pool, 0);
+ if (addr)
+ return addr;
+
+ addr = pop_hyp_memcache(mc, hyp_phys_to_virt);
+ if (!addr)
+ return addr;
+
+ memset(addr, 0, PAGE_SIZE);
+ p = hyp_virt_to_page(addr);
+ memset(p, 0, sizeof(*p));
+ p->refcount = 1;
+
+ return addr;
+}
+
+static void guest_s2_get_page(void *addr)
+{
+ hyp_get_page(¤t_vm->pool, addr);
+}
+
+static void guest_s2_put_page(void *addr)
+{
+ hyp_put_page(¤t_vm->pool, addr);
+}
+
+static void clean_dcache_guest_page(void *va, size_t size)
+{
+ __clean_dcache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
+ hyp_fixmap_unmap();
+}
+
+static void invalidate_icache_guest_page(void *va, size_t size)
+{
+ __invalidate_icache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
+ hyp_fixmap_unmap();
+}
+
int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd)
{
- vm->pgt.pgd = pgd;
+ struct kvm_s2_mmu *mmu = &vm->kvm.arch.mmu;
+ unsigned long nr_pages;
+ int ret;
+
+ nr_pages = kvm_pgtable_stage2_pgd_size(vm->kvm.arch.vtcr) >> PAGE_SHIFT;
+ ret = hyp_pool_init(&vm->pool, hyp_virt_to_pfn(pgd), nr_pages, 0);
+ if (ret)
+ return ret;
+
+ hyp_spin_lock_init(&vm->lock);
+ vm->mm_ops = (struct kvm_pgtable_mm_ops) {
+ .zalloc_pages_exact = guest_s2_zalloc_pages_exact,
+ .free_pages_exact = guest_s2_free_pages_exact,
+ .zalloc_page = guest_s2_zalloc_page,
+ .phys_to_virt = hyp_phys_to_virt,
+ .virt_to_phys = hyp_virt_to_phys,
+ .page_count = hyp_page_count,
+ .get_page = guest_s2_get_page,
+ .put_page = guest_s2_put_page,
+ .dcache_clean_inval_poc = clean_dcache_guest_page,
+ .icache_inval_pou = invalidate_icache_guest_page,
+ };
+
+ guest_lock_component(vm);
+ ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0,
+ guest_stage2_force_pte_cb);
+ guest_unlock_component(vm);
+ if (ret)
+ return ret;
+
+ vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
+
return 0;
}
void reclaim_guest_pages(struct pkvm_hyp_vm *vm)
{
+ void *pgd = vm->pgt.pgd;
unsigned long nr_pages;
nr_pages = kvm_pgtable_stage2_pgd_size(vm->kvm.arch.vtcr) >> PAGE_SHIFT;
- WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(vm->pgt.pgd), nr_pages));
+
+ guest_lock_component(vm);
+ kvm_pgtable_stage2_destroy(&vm->pgt);
+ vm->kvm.arch.mmu.pgd_phys = 0ULL;
+ guest_unlock_component(vm);
+
+ WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(pgd), nr_pages));
}
int __pkvm_prot_finalize(void)
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 43761d31f763..301fc7275062 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -686,7 +686,9 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t
return -EINVAL;
phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
- if (phys_shift) {
+ if (is_protected_kvm_enabled()) {
+ phys_shift = kvm_ipa_limit;
+ } else if (phys_shift) {
if (phys_shift > kvm_ipa_limit ||
phys_shift < ARM64_MIN_PARANGE_BITS)
return -EINVAL;
--
2.38.0.413.g74048e4d9e-goog
next prev parent reply other threads:[~2022-10-17 11:53 UTC|newest]
Thread overview: 53+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-17 11:51 [PATCH v4 00/25] KVM: arm64: Introduce pKVM hyp VM and vCPU state at EL2 Will Deacon
2022-10-17 11:51 ` [PATCH v4 01/25] KVM: arm64: Move hyp refcount manipulation helpers to common header file Will Deacon
2022-10-17 20:29 ` Philippe Mathieu-Daudé
2022-10-17 11:51 ` [PATCH v4 02/25] KVM: arm64: Allow attaching of non-coalescable pages to a hyp pool Will Deacon
2022-10-17 11:51 ` [PATCH v4 03/25] KVM: arm64: Back the hypervisor 'struct hyp_page' array for all memory Will Deacon
2022-10-17 11:51 ` [PATCH v4 04/25] KVM: arm64: Fix-up hyp stage-1 refcounts for all pages mapped at EL2 Will Deacon
2022-10-17 11:51 ` [PATCH v4 05/25] KVM: arm64: Unify identifiers used to distinguish host and hypervisor Will Deacon
2022-10-17 20:21 ` Philippe Mathieu-Daudé
2022-10-17 11:51 ` [PATCH v4 06/25] KVM: arm64: Implement do_donate() helper for donating memory Will Deacon
2022-10-17 11:51 ` [PATCH v4 07/25] KVM: arm64: Prevent the donation of no-map pages Will Deacon
2022-10-18 13:42 ` Philippe Mathieu-Daudé
2022-10-17 11:51 ` [PATCH v4 08/25] KVM: arm64: Add helpers to pin memory shared with the hypervisor at EL2 Will Deacon
2022-10-17 11:51 ` [PATCH v4 09/25] KVM: arm64: Include asm/kvm_mmu.h in nvhe/mem_protect.h Will Deacon
2022-10-17 20:22 ` Philippe Mathieu-Daudé
2022-10-17 11:51 ` [PATCH v4 10/25] KVM: arm64: Add hyp_spinlock_t static initializer Will Deacon
2022-10-18 13:51 ` Philippe Mathieu-Daudé
2022-10-17 11:51 ` [PATCH v4 11/25] KVM: arm64: Rename 'host_kvm' to 'host_mmu' Will Deacon
2022-10-18 13:47 ` Philippe Mathieu-Daudé
2022-10-17 11:51 ` [PATCH v4 12/25] KVM: arm64: Add infrastructure to create and track pKVM instances at EL2 Will Deacon
2022-10-18 15:13 ` Quentin Perret
2022-10-19 12:35 ` Will Deacon
2022-10-18 16:21 ` Quentin Perret
2022-10-19 12:45 ` Will Deacon
2022-10-18 16:33 ` Quentin Perret
2022-10-19 11:57 ` Will Deacon
2022-10-19 13:35 ` Quentin Perret
2022-10-18 16:40 ` Quentin Perret
2022-10-19 12:44 ` Will Deacon
2022-10-18 16:45 ` Quentin Perret
2022-10-19 12:18 ` Fuad Tabba
2022-10-17 11:51 ` [PATCH v4 13/25] KVM: arm64: Instantiate pKVM hypervisor VM and vCPU structures from EL1 Will Deacon
2022-10-19 15:46 ` Quentin Perret
2022-10-19 16:00 ` Quentin Perret
2022-10-19 16:34 ` Will Deacon
2022-10-17 11:51 ` [PATCH v4 14/25] KVM: arm64: Add per-cpu fixmap infrastructure at EL2 Will Deacon
2022-10-18 11:06 ` Mark Rutland
2022-10-18 14:05 ` Will Deacon
2022-10-18 16:52 ` Mark Rutland
2022-10-19 12:01 ` Will Deacon
2022-10-17 11:51 ` [PATCH v4 15/25] KVM: arm64: Initialise hypervisor copies of host symbols unconditionally Will Deacon
2022-10-17 20:26 ` Philippe Mathieu-Daudé
2022-10-17 11:52 ` [PATCH v4 16/25] KVM: arm64: Provide I-cache invalidation by virtual address at EL2 Will Deacon
2022-10-17 11:52 ` [PATCH v4 17/25] KVM: arm64: Add generic hyp_memcache helpers Will Deacon
2022-10-17 11:52 ` [PATCH v4 18/25] KVM: arm64: Consolidate stage-2 initialisation into a single function Will Deacon
2022-10-17 11:52 ` Will Deacon [this message]
2022-10-17 11:52 ` [PATCH v4 20/25] KVM: arm64: Return guest memory from EL2 via dedicated teardown memcache Will Deacon
2022-10-19 15:52 ` Quentin Perret
2022-10-19 16:24 ` Will Deacon
2022-10-17 11:52 ` [PATCH v4 21/25] KVM: arm64: Unmap 'kvm_arm_hyp_percpu_base' from the host Will Deacon
2022-10-17 11:52 ` [PATCH v4 22/25] KVM: arm64: Maintain a copy of 'kvm_arm_vmid_bits' at EL2 Will Deacon
2022-10-17 11:52 ` [PATCH v4 23/25] KVM: arm64: Explicitly map 'kvm_vgic_global_state' " Will Deacon
2022-10-17 11:52 ` [PATCH v4 24/25] KVM: arm64: Don't unnecessarily map host kernel sections " Will Deacon
2022-10-17 11:52 ` [PATCH v4 25/25] KVM: arm64: Use the pKVM hyp vCPU structure in handle___kvm_vcpu_run() Will Deacon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221017115209.2099-20-will@kernel.org \
--to=will@kernel.org \
--cc=alexandru.elisei@arm.com \
--cc=catalin.marinas@arm.com \
--cc=chao.p.peng@linux.intel.com \
--cc=james.morse@arm.com \
--cc=kernel-team@android.com \
--cc=kvm@vger.kernel.org \
--cc=kvmarm@lists.linux.dev \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=mark.rutland@arm.com \
--cc=maz@kernel.org \
--cc=oliver.upton@linux.dev \
--cc=qperret@google.com \
--cc=seanjc@google.com \
--cc=suzuki.poulose@arm.com \
--cc=tabba@google.com \
--cc=vdonnefort@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).