From: Quentin Perret <qperret@google.com>
To: catalin.marinas@arm.com, will@kernel.org, maz@kernel.org,
james.morse@arm.com, julien.thierry.kdev@gmail.com,
suzuki.poulose@arm.com
Cc: android-kvm@google.com, linux-kernel@vger.kernel.org,
kernel-team@android.com, kvmarm@lists.cs.columbia.edu,
linux-arm-kernel@lists.infradead.org, tabba@google.com,
mark.rutland@arm.com, dbrazdil@google.com, mate.toth-pal@arm.com,
seanjc@google.com, qperret@google.com, robh+dt@kernel.org
Subject: [PATCH v3 12/32] KVM: arm64: Introduce a Hyp buddy page allocator
Date: Tue, 2 Mar 2021 14:59:42 +0000 [thread overview]
Message-ID: <20210302150002.3685113-13-qperret@google.com> (raw)
In-Reply-To: <20210302150002.3685113-1-qperret@google.com>
When memory protection is enabled, the hyp code will require a basic
form of memory management in order to allocate and free memory pages at
EL2. This is needed for various use-cases, including the creation of hyp
mappings or the allocation of stage 2 page tables.
To address these use-case, introduce a simple memory allocator in the
hyp code. The allocator is designed as a conventional 'buddy allocator',
working with a page granularity. It allows to allocate and free
physically contiguous pages from memory 'pools', with a guaranteed order
alignment in the PA space. Each page in a memory pool is associated
with a struct hyp_page which holds the page's metadata, including its
refcount, as well as its current order, hence mimicking the kernel's
buddy system in the GFP infrastructure. The hyp_page metadata are made
accessible through a hyp_vmemmap, following the concept of
SPARSE_VMEMMAP in the kernel.
Signed-off-by: Quentin Perret <qperret@google.com>
---
arch/arm64/kvm/hyp/include/nvhe/gfp.h | 55 +++++++
arch/arm64/kvm/hyp/include/nvhe/memory.h | 28 ++++
arch/arm64/kvm/hyp/nvhe/Makefile | 2 +-
arch/arm64/kvm/hyp/nvhe/page_alloc.c | 195 +++++++++++++++++++++++
4 files changed, 279 insertions(+), 1 deletion(-)
create mode 100644 arch/arm64/kvm/hyp/include/nvhe/gfp.h
create mode 100644 arch/arm64/kvm/hyp/nvhe/page_alloc.c
diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h b/arch/arm64/kvm/hyp/include/nvhe/gfp.h
new file mode 100644
index 000000000000..d039086d86b5
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __KVM_HYP_GFP_H
+#define __KVM_HYP_GFP_H
+
+#include <linux/list.h>
+
+#include <nvhe/memory.h>
+#include <nvhe/spinlock.h>
+
+#define HYP_NO_ORDER UINT_MAX
+
+struct hyp_pool {
+ /*
+ * Spinlock protecting concurrent changes to the memory pool as well as
+ * the struct hyp_page of the pool's pages until we have a proper atomic
+ * API at EL2.
+ */
+ hyp_spinlock_t lock;
+ struct list_head free_area[MAX_ORDER];
+ phys_addr_t range_start;
+ phys_addr_t range_end;
+ unsigned int max_order;
+};
+
+static inline void hyp_page_ref_inc(struct hyp_page *p)
+{
+ struct hyp_pool *pool = hyp_page_to_pool(p);
+
+ hyp_spin_lock(&pool->lock);
+ p->refcount++;
+ hyp_spin_unlock(&pool->lock);
+}
+
+static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
+{
+ struct hyp_pool *pool = hyp_page_to_pool(p);
+ int ret;
+
+ hyp_spin_lock(&pool->lock);
+ p->refcount--;
+ ret = (p->refcount == 0);
+ hyp_spin_unlock(&pool->lock);
+
+ return ret;
+}
+
+/* Allocation */
+void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order);
+void hyp_get_page(void *addr);
+void hyp_put_page(void *addr);
+
+/* Used pages cannot be freed */
+int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages,
+ unsigned int reserved_pages);
+#endif /* __KVM_HYP_GFP_H */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/include/nvhe/memory.h
index 3e49eaa7e682..d2fb307c5952 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/memory.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h
@@ -6,7 +6,17 @@
#include <linux/types.h>
+struct hyp_pool;
+struct hyp_page {
+ unsigned int refcount;
+ unsigned int order;
+ struct hyp_pool *pool;
+ struct list_head node;
+};
+
extern s64 hyp_physvirt_offset;
+extern u64 __hyp_vmemmap;
+#define hyp_vmemmap ((struct hyp_page *)__hyp_vmemmap)
#define __hyp_pa(virt) ((phys_addr_t)(virt) + hyp_physvirt_offset)
#define __hyp_va(phys) ((void *)((phys_addr_t)(phys) - hyp_physvirt_offset))
@@ -21,4 +31,22 @@ static inline phys_addr_t hyp_virt_to_phys(void *addr)
return __hyp_pa(addr);
}
+#define hyp_phys_to_pfn(phys) ((phys) >> PAGE_SHIFT)
+#define hyp_pfn_to_phys(pfn) ((phys_addr_t)((pfn) << PAGE_SHIFT))
+#define hyp_phys_to_page(phys) (&hyp_vmemmap[hyp_phys_to_pfn(phys)])
+#define hyp_virt_to_page(virt) hyp_phys_to_page(__hyp_pa(virt))
+#define hyp_virt_to_pfn(virt) hyp_phys_to_pfn(__hyp_pa(virt))
+
+#define hyp_page_to_pfn(page) ((struct hyp_page *)(page) - hyp_vmemmap)
+#define hyp_page_to_phys(page) hyp_pfn_to_phys((hyp_page_to_pfn(page)))
+#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page))
+#define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool)
+
+static inline int hyp_page_count(void *addr)
+{
+ struct hyp_page *p = hyp_virt_to_page(addr);
+
+ return p->refcount;
+}
+
#endif /* __KVM_HYP_MEMORY_H */
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 144da72ad510..6894a917f290 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -13,7 +13,7 @@ lib-objs := clear_page.o copy_page.o memcpy.o memset.o
lib-objs := $(addprefix ../../../lib/, $(lib-objs))
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
- hyp-main.o hyp-smp.o psci-relay.o early_alloc.o stub.o
+ hyp-main.o hyp-smp.o psci-relay.o early_alloc.o stub.o page_alloc.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o
obj-y += $(lib-objs)
diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c
new file mode 100644
index 000000000000..348fc4329e8f
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c
@@ -0,0 +1,195 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Google LLC
+ * Author: Quentin Perret <qperret@google.com>
+ */
+
+#include <asm/kvm_hyp.h>
+#include <nvhe/gfp.h>
+
+u64 __hyp_vmemmap;
+
+/*
+ * Index the hyp_vmemmap to find a potential buddy page, but make no assumption
+ * about its current state.
+ *
+ * Example buddy-tree for a 4-pages physically contiguous pool:
+ *
+ * o : Page 3
+ * /
+ * o-o : Page 2
+ * /
+ * / o : Page 1
+ * / /
+ * o---o-o : Page 0
+ * Order 2 1 0
+ *
+ * Example of requests on this pool:
+ * __find_buddy_nocheck(pool, page 0, order 0) => page 1
+ * __find_buddy_nocheck(pool, page 0, order 1) => page 2
+ * __find_buddy_nocheck(pool, page 1, order 0) => page 0
+ * __find_buddy_nocheck(pool, page 2, order 0) => page 3
+ */
+static struct hyp_page *__find_buddy_nocheck(struct hyp_pool *pool,
+ struct hyp_page *p,
+ unsigned int order)
+{
+ phys_addr_t addr = hyp_page_to_phys(p);
+
+ addr ^= (PAGE_SIZE << order);
+
+ /*
+ * Don't return a page outside the pool range -- it belongs to
+ * something else and may not be mapped in hyp_vmemmap.
+ */
+ if (addr < pool->range_start || addr >= pool->range_end)
+ return NULL;
+
+ return hyp_phys_to_page(addr);
+}
+
+/* Find a buddy page currently available for allocation */
+static struct hyp_page *__find_buddy_avail(struct hyp_pool *pool,
+ struct hyp_page *p,
+ unsigned int order)
+{
+ struct hyp_page *buddy = __find_buddy_nocheck(pool, p, order);
+
+ if (!buddy || buddy->order != order || list_empty(&buddy->node))
+ return NULL;
+
+ return buddy;
+
+}
+
+static void __hyp_attach_page(struct hyp_pool *pool,
+ struct hyp_page *p)
+{
+ unsigned int order = p->order;
+ struct hyp_page *buddy;
+
+ memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);
+
+ /*
+ * Only the first struct hyp_page of a high-order page (otherwise known
+ * as the 'head') should have p->order set. The non-head pages should
+ * have p->order = HYP_NO_ORDER. Here @p may no longer be the head
+ * after coallescing, so make sure to mark it HYP_NO_ORDER proactively.
+ */
+ p->order = HYP_NO_ORDER;
+ for (; (order + 1) < pool->max_order; order++) {
+ buddy = __find_buddy_avail(pool, p, order);
+ if (!buddy)
+ break;
+
+ /* Take the buddy out of its list, and coallesce with @p */
+ list_del_init(&buddy->node);
+ buddy->order = HYP_NO_ORDER;
+ p = (p < buddy) ? p : buddy;
+ }
+
+ /* Mark the new head, and insert it */
+ p->order = order;
+ list_add_tail(&p->node, &pool->free_area[order]);
+}
+
+static void hyp_attach_page(struct hyp_page *p)
+{
+ struct hyp_pool *pool = hyp_page_to_pool(p);
+
+ hyp_spin_lock(&pool->lock);
+ __hyp_attach_page(pool, p);
+ hyp_spin_unlock(&pool->lock);
+}
+
+static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool,
+ struct hyp_page *p,
+ unsigned int order)
+{
+ struct hyp_page *buddy;
+
+ list_del_init(&p->node);
+ while (p->order > order) {
+ /*
+ * The buddy of order n - 1 currently has HYP_NO_ORDER as it
+ * is covered by a higher-level page (whose head is @p). Use
+ * __find_buddy_nocheck() to find it and inject it in the
+ * free_list[n - 1], effectively splitting @p in half.
+ */
+ p->order--;
+ buddy = __find_buddy_nocheck(pool, p, p->order);
+ buddy->order = p->order;
+ list_add_tail(&buddy->node, &pool->free_area[buddy->order]);
+ }
+
+ return p;
+}
+
+void hyp_put_page(void *addr)
+{
+ struct hyp_page *p = hyp_virt_to_page(addr);
+
+ if (hyp_page_ref_dec_and_test(p))
+ hyp_attach_page(p);
+}
+
+void hyp_get_page(void *addr)
+{
+ struct hyp_page *p = hyp_virt_to_page(addr);
+
+ hyp_page_ref_inc(p);
+}
+
+void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order)
+{
+ unsigned int i = order;
+ struct hyp_page *p;
+
+ hyp_spin_lock(&pool->lock);
+
+ /* Look for a high-enough-order page */
+ while (i < pool->max_order && list_empty(&pool->free_area[i]))
+ i++;
+ if (i >= pool->max_order) {
+ hyp_spin_unlock(&pool->lock);
+ return NULL;
+ }
+
+ /* Extract it from the tree at the right order */
+ p = list_first_entry(&pool->free_area[i], struct hyp_page, node);
+ p = __hyp_extract_page(pool, p, order);
+
+ hyp_spin_unlock(&pool->lock);
+ hyp_page_ref_inc(p);
+
+ return p ? hyp_page_to_virt(p) : NULL;
+}
+
+int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages,
+ unsigned int reserved_pages)
+{
+ phys_addr_t phys = hyp_pfn_to_phys(pfn);
+ struct hyp_page *p;
+ int i;
+
+ hyp_spin_lock_init(&pool->lock);
+ pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT));
+ for (i = 0; i < pool->max_order; i++)
+ INIT_LIST_HEAD(&pool->free_area[i]);
+ pool->range_start = phys;
+ pool->range_end = phys + (nr_pages << PAGE_SHIFT);
+
+ /* Init the vmemmap portion */
+ p = hyp_phys_to_page(phys);
+ memset(p, 0, sizeof(*p) * nr_pages);
+ for (i = 0; i < nr_pages; i++) {
+ p[i].pool = pool;
+ INIT_LIST_HEAD(&p[i].node);
+ }
+
+ /* Attach the unused pages to the buddy tree */
+ for (i = reserved_pages; i < nr_pages; i++)
+ __hyp_attach_page(pool, &p[i]);
+
+ return 0;
+}
--
2.30.1.766.gb4fecdf3b7-goog
next prev parent reply other threads:[~2021-03-02 17:22 UTC|newest]
Thread overview: 64+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-02 14:59 [PATCH v3 00/32] KVM: arm64: A stage 2 for the host Quentin Perret
2021-03-02 14:59 ` [PATCH v3 01/32] arm64: lib: Annotate {clear,copy}_page() as position-independent Quentin Perret
2021-03-02 14:59 ` [PATCH v3 02/32] KVM: arm64: Link position-independent string routines into .hyp.text Quentin Perret
2021-03-02 14:59 ` [PATCH v3 03/32] arm64: kvm: Add standalone ticket spinlock implementation for use at hyp Quentin Perret
2021-03-02 14:59 ` [PATCH v3 04/32] KVM: arm64: Initialize kvm_nvhe_init_params early Quentin Perret
2021-03-04 13:39 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 05/32] KVM: arm64: Avoid free_page() in page-table allocator Quentin Perret
2021-03-02 14:59 ` [PATCH v3 06/32] KVM: arm64: Factor memory allocation out of pgtable.c Quentin Perret
2021-03-04 14:06 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 07/32] KVM: arm64: Introduce a BSS section for use at Hyp Quentin Perret
2021-03-04 14:09 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 08/32] KVM: arm64: Make kvm_call_hyp() a function call " Quentin Perret
2021-03-02 14:59 ` [PATCH v3 09/32] KVM: arm64: Allow using kvm_nvhe_sym() in hyp code Quentin Perret
2021-03-02 14:59 ` [PATCH v3 10/32] KVM: arm64: Introduce an early Hyp page allocator Quentin Perret
2021-03-04 14:38 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 11/32] KVM: arm64: Stub CONFIG_DEBUG_LIST at Hyp Quentin Perret
2021-03-02 14:59 ` Quentin Perret [this message]
2021-03-04 15:30 ` [PATCH v3 12/32] KVM: arm64: Introduce a Hyp buddy page allocator Will Deacon
2021-03-04 15:49 ` Quentin Perret
2021-03-02 14:59 ` [PATCH v3 13/32] KVM: arm64: Enable access to sanitized CPU features at EL2 Quentin Perret
2021-03-02 14:59 ` [PATCH v3 14/32] KVM: arm64: Factor out vector address calculation Quentin Perret
2021-03-02 14:59 ` [PATCH v3 15/32] KVM: arm64: Prepare the creation of s1 mappings at EL2 Quentin Perret
2021-03-04 18:47 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 16/32] KVM: arm64: Elevate hypervisor mappings creation " Quentin Perret
2021-03-04 19:25 ` Will Deacon
2021-03-05 9:14 ` Quentin Perret
2021-03-02 14:59 ` [PATCH v3 17/32] KVM: arm64: Use kvm_arch for stage 2 pgtable Quentin Perret
2021-03-02 14:59 ` [PATCH v3 18/32] KVM: arm64: Use kvm_arch in kvm_s2_mmu Quentin Perret
2021-03-02 14:59 ` [PATCH v3 19/32] KVM: arm64: Set host stage 2 using kvm_nvhe_init_params Quentin Perret
2021-03-02 14:59 ` [PATCH v3 20/32] KVM: arm64: Refactor kvm_arm_setup_stage2() Quentin Perret
2021-03-04 19:35 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 21/32] KVM: arm64: Refactor __load_guest_stage2() Quentin Perret
2021-03-02 14:59 ` [PATCH v3 22/32] KVM: arm64: Refactor __populate_fault_info() Quentin Perret
2021-03-04 19:39 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 23/32] KVM: arm64: Make memcache anonymous in pgtable allocator Quentin Perret
2021-03-04 19:44 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 24/32] KVM: arm64: Reserve memory for host stage 2 Quentin Perret
2021-03-04 19:49 ` Will Deacon
2021-03-05 9:17 ` Quentin Perret
2021-03-02 14:59 ` [PATCH v3 25/32] KVM: arm64: Sort the hypervisor memblocks Quentin Perret
2021-03-04 19:51 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 26/32] KVM: arm64: Introduce PROT_NONE mappings for stage 2 Quentin Perret
2021-03-04 20:00 ` Will Deacon
2021-03-05 9:52 ` Quentin Perret
2021-03-05 19:03 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 27/32] KVM: arm64: Refactor stage2_map_set_prot_attr() Quentin Perret
2021-03-04 20:03 ` Will Deacon
2021-03-05 9:18 ` Quentin Perret
2021-03-02 14:59 ` [PATCH v3 28/32] KVM: arm64: Add kvm_pgtable_stage2_idmap_greedy() Quentin Perret
2021-03-05 14:39 ` Will Deacon
2021-03-05 15:03 ` Quentin Perret
2021-03-05 16:59 ` Will Deacon
2021-03-02 14:59 ` [PATCH v3 29/32] KVM: arm64: Wrap the host with a stage 2 Quentin Perret
2021-03-05 19:29 ` Will Deacon
2021-03-08 9:22 ` Quentin Perret
2021-03-08 12:46 ` Will Deacon
2021-03-08 13:38 ` Quentin Perret
2021-03-08 13:52 ` Will Deacon
2021-03-02 15:00 ` [PATCH v3 30/32] KVM: arm64: Page-align the .hyp sections Quentin Perret
2021-03-04 20:05 ` Will Deacon
2021-03-02 15:00 ` [PATCH v3 31/32] KVM: arm64: Disable PMU support in protected mode Quentin Perret
2021-03-05 19:02 ` Will Deacon
2021-03-02 15:00 ` [PATCH v3 32/32] KVM: arm64: Protect the .hyp sections from the host Quentin Perret
2021-03-05 19:13 ` Will Deacon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210302150002.3685113-13-qperret@google.com \
--to=qperret@google.com \
--cc=android-kvm@google.com \
--cc=catalin.marinas@arm.com \
--cc=dbrazdil@google.com \
--cc=james.morse@arm.com \
--cc=julien.thierry.kdev@gmail.com \
--cc=kernel-team@android.com \
--cc=kvmarm@lists.cs.columbia.edu \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mate.toth-pal@arm.com \
--cc=maz@kernel.org \
--cc=robh+dt@kernel.org \
--cc=seanjc@google.com \
--cc=suzuki.poulose@arm.com \
--cc=tabba@google.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).