From: Mark Rutland <mark.rutland@arm.com>
To: linux-arm-kernel@lists.infradead.org
Cc: alexandru.elisei@arm.com, andrii@kernel.org, ardb@kernel.org,
ast@kernel.org, broonie@kernel.org, catalin.marinas@arm.com,
daniel@iogearbox.net, dvyukov@google.com, james.morse@arm.com,
jean-philippe@linaro.org, jpoimboe@redhat.com,
mark.rutland@arm.com, maz@kernel.org, peterz@infradead.org,
robin.murphy@arm.com, suzuki.poulose@arm.com, will@kernel.org
Subject: [PATCH v2 10/13] arm64: extable: add `type` and `data` fields
Date: Tue, 19 Oct 2021 17:02:16 +0100 [thread overview]
Message-ID: <20211019160219.5202-11-mark.rutland@arm.com> (raw)
In-Reply-To: <20211019160219.5202-1-mark.rutland@arm.com>
Subsequent patches will add specialized handlers for fixups, in addition
to the simple PC fixup and BPF handlers we have today. In preparation,
this patch adds a new `type` field to struct exception_table_entry, and
uses this to distinguish the fixup and BPF cases. A `data` field is also
added so that subsequent patches can associate data specific to each
exception site (e.g. register numbers).
Handlers are named ex_handler_*() for consistency, following the exmaple
of x86. At the same time, get_ex_fixup() is split out into a helper so
that it can be used by other ex_handler_*() functions ins subsequent
patches.
This patch will increase the size of the exception tables, which will be
remedied by subsequent patches removing redundant fixup code. There
should be no functional change as a result of this patch.
Since each entry is now 12 bytes in size, we must reduce the alignment
of each entry from `.align 3` (i.e. 8 bytes) to `.align 2` (i.e. 4
bytes), which is the natrual alignment of the `insn` and `fixup` fields.
The current 8-byte alignment is a holdover from when the `insn` and
`fixup` fields was 8 bytes, and while not harmful has not been necessary
since commit:
6c94f27ac847ff8e ("arm64: switch to relative exception tables")
Similarly, RO_EXCEPTION_TABLE_ALIGN is dropped to 4 bytes.
Concurrently with this patch, x86's exception table entry format is
being updated (similarly to a 12-byte format, with 32-bytes of absolute
data). Once both have been merged it should be possible to unify the
sorttable logic for the two.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: James Morse <james.morse@arm.com>
Cc: Jean-Philippe Brucker <jean-philippe@linaro.org>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/asm-extable.h | 32 ++++++++++++++++++++------------
arch/arm64/include/asm/extable.h | 19 +++++++++++++++----
arch/arm64/kernel/vmlinux.lds.S | 2 +-
arch/arm64/mm/extable.c | 29 +++++++++++++++++++++++++----
arch/arm64/net/bpf_jit_comp.c | 7 +++++--
scripts/sorttable.c | 30 ++++++++++++++++++++++++++++++
6 files changed, 96 insertions(+), 23 deletions(-)
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index 986b4c0d4792..5ee748edaef1 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -2,13 +2,19 @@
#ifndef __ASM_ASM_EXTABLE_H
#define __ASM_ASM_EXTABLE_H
+#define EX_TYPE_NONE 0
+#define EX_TYPE_FIXUP 1
+#define EX_TYPE_BPF 2
+
#ifdef __ASSEMBLY__
-#define __ASM_EXTABLE_RAW(insn, fixup) \
- .pushsection __ex_table, "a"; \
- .align 3; \
- .long ((insn) - .); \
- .long ((fixup) - .); \
+#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
+ .pushsection __ex_table, "a"; \
+ .align 2; \
+ .long ((insn) - .); \
+ .long ((fixup) - .); \
+ .short (type); \
+ .short (data); \
.popsection;
/*
@@ -16,7 +22,7 @@
* when an unhandled fault is taken.
*/
.macro _asm_extable, insn, fixup
- __ASM_EXTABLE_RAW(\insn, \fixup)
+ __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
.endm
/*
@@ -33,15 +39,17 @@
#include <linux/stringify.h>
-#define __ASM_EXTABLE_RAW(insn, fixup) \
- ".pushsection __ex_table, \"a\"\n" \
- ".align 3\n" \
- ".long ((" insn ") - .)\n" \
- ".long ((" fixup ") - .)\n" \
+#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
+ ".pushsection __ex_table, \"a\"\n" \
+ ".align 2\n" \
+ ".long ((" insn ") - .)\n" \
+ ".long ((" fixup ") - .)\n" \
+ ".short (" type ")\n" \
+ ".short (" data ")\n" \
".popsection\n"
#define _ASM_EXTABLE(insn, fixup) \
- __ASM_EXTABLE_RAW(#insn, #fixup)
+ __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
#endif /* __ASSEMBLY__ */
diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
index 1859b9fd566f..8b300dd28def 100644
--- a/arch/arm64/include/asm/extable.h
+++ b/arch/arm64/include/asm/extable.h
@@ -18,10 +18,21 @@
struct exception_table_entry
{
int insn, fixup;
+ short type, data;
};
#define ARCH_HAS_RELATIVE_EXTABLE
+#define swap_ex_entry_fixup(a, b, tmp, delta) \
+do { \
+ (a)->fixup = (b)->fixup + (delta); \
+ (b)->fixup = (tmp).fixup - (delta); \
+ (a)->type = (b)->type; \
+ (b)->type = (tmp).type; \
+ (a)->data = (b)->data; \
+ (b)->data = (tmp).data; \
+} while (0)
+
static inline bool in_bpf_jit(struct pt_regs *regs)
{
if (!IS_ENABLED(CONFIG_BPF_JIT))
@@ -32,12 +43,12 @@ static inline bool in_bpf_jit(struct pt_regs *regs)
}
#ifdef CONFIG_BPF_JIT
-bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
- struct pt_regs *regs);
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+ struct pt_regs *regs);
#else /* !CONFIG_BPF_JIT */
static inline
-bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
- struct pt_regs *regs)
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
{
return false;
}
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index f6b1a88245db..fa8a8e8ddcfd 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -57,7 +57,7 @@
#define SBSS_ALIGN 0
#endif
-#define RO_EXCEPTION_TABLE_ALIGN 8
+#define RO_EXCEPTION_TABLE_ALIGN 4
#define RUNTIME_DISCARD_EXIT
#include <asm-generic/vmlinux.lds.h>
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index dba3d59f3eca..c2951b963335 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -6,6 +6,24 @@
#include <linux/extable.h>
#include <linux/uaccess.h>
+#include <asm/asm-extable.h>
+
+typedef bool (*ex_handler_t)(const struct exception_table_entry *,
+ struct pt_regs *);
+
+static inline unsigned long
+get_ex_fixup(const struct exception_table_entry *ex)
+{
+ return ((unsigned long)&ex->fixup + ex->fixup);
+}
+
+static bool ex_handler_fixup(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
+{
+ regs->pc = get_ex_fixup(ex);
+ return true;
+}
+
bool fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *ex;
@@ -14,9 +32,12 @@ bool fixup_exception(struct pt_regs *regs)
if (!ex)
return false;
- if (in_bpf_jit(regs))
- return arm64_bpf_fixup_exception(ex, regs);
+ switch (ex->type) {
+ case EX_TYPE_FIXUP:
+ return ex_handler_fixup(ex, regs);
+ case EX_TYPE_BPF:
+ return ex_handler_bpf(ex, regs);
+ }
- regs->pc = (unsigned long)&ex->fixup + ex->fixup;
- return true;
+ BUG();
}
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 956c841ef346..7df7345e60d8 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -13,6 +13,7 @@
#include <linux/printk.h>
#include <linux/slab.h>
+#include <asm/asm-extable.h>
#include <asm/byteorder.h>
#include <asm/cacheflush.h>
#include <asm/debug-monitors.h>
@@ -358,8 +359,8 @@ static void build_epilogue(struct jit_ctx *ctx)
#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
-bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
- struct pt_regs *regs)
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
{
off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
@@ -412,6 +413,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
+ ex->type = EX_TYPE_BPF;
+
ctx->exentry_idx++;
return 0;
}
diff --git a/scripts/sorttable.c b/scripts/sorttable.c
index 6ee4fa882919..ee95bb47a50d 100644
--- a/scripts/sorttable.c
+++ b/scripts/sorttable.c
@@ -231,6 +231,34 @@ static void sort_relative_table(char *extab_image, int image_size)
}
}
+static void arm64_sort_relative_table(char *extab_image, int image_size)
+{
+ int i = 0;
+
+ while (i < image_size) {
+ uint32_t *loc = (uint32_t *)(extab_image + i);
+
+ w(r(loc) + i, loc);
+ w(r(loc + 1) + i + 4, loc + 1);
+ /* Don't touch the fixup type or data */
+
+ i += sizeof(uint32_t) * 3;
+ }
+
+ qsort(extab_image, image_size / 12, 12, compare_relative_table);
+
+ i = 0;
+ while (i < image_size) {
+ uint32_t *loc = (uint32_t *)(extab_image + i);
+
+ w(r(loc) - i, loc);
+ w(r(loc + 1) - (i + 4), loc + 1);
+ /* Don't touch the fixup type or data */
+
+ i += sizeof(uint32_t) * 3;
+ }
+}
+
static void x86_sort_relative_table(char *extab_image, int image_size)
{
int i = 0;
@@ -343,6 +371,8 @@ static int do_file(char const *const fname, void *addr)
custom_sort = s390_sort_relative_table;
break;
case EM_AARCH64:
+ custom_sort = arm64_sort_relative_table;
+ break;
case EM_PARISC:
case EM_PPC:
case EM_PPC64:
--
2.11.0
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2021-10-19 16:06 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-19 16:02 [PATCH v2 00/13] arm64: extable: remove anonymous out-of-line fixups Mark Rutland
2021-10-19 16:02 ` [PATCH v2 01/13] arm64: lib: __arch_clear_user(): fold fixups into body Mark Rutland
2021-10-19 16:02 ` [PATCH v2 02/13] arm64: lib: __arch_copy_from_user(): " Mark Rutland
2021-10-19 16:02 ` [PATCH v2 03/13] arm64: lib: __arch_copy_to_user(): " Mark Rutland
2021-10-19 16:02 ` [PATCH v2 04/13] arm64: kvm: use kvm_exception_table_entry Mark Rutland
2021-10-21 9:09 ` Marc Zyngier
2021-10-19 16:02 ` [PATCH v2 05/13] arm64: factor out GPR numbering helpers Mark Rutland
2021-10-19 16:02 ` [PATCH v2 06/13] arm64: gpr-num: support W registers Mark Rutland
2021-10-19 16:02 ` [PATCH v2 07/13] arm64: extable: consolidate definitions Mark Rutland
2021-10-19 16:02 ` [PATCH v2 08/13] arm64: extable: make fixup_exception() return bool Mark Rutland
2021-10-19 16:02 ` [PATCH v2 09/13] arm64: extable: use `ex` for `exception_table_entry` Mark Rutland
2021-10-19 16:02 ` Mark Rutland [this message]
2021-10-19 16:02 ` [PATCH v2 11/13] arm64: extable: add a dedicated uaccess handler Mark Rutland
2021-10-19 16:02 ` [PATCH v2 12/13] arm64: extable: add load_unaligned_zeropad() handler Mark Rutland
2021-10-19 16:02 ` [PATCH v2 13/13] arm64: vmlinux.lds.S: remove `.fixup` section Mark Rutland
2021-10-21 10:05 ` [PATCH v2 00/13] arm64: extable: remove anonymous out-of-line fixups Will Deacon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211019160219.5202-11-mark.rutland@arm.com \
--to=mark.rutland@arm.com \
--cc=alexandru.elisei@arm.com \
--cc=andrii@kernel.org \
--cc=ardb@kernel.org \
--cc=ast@kernel.org \
--cc=broonie@kernel.org \
--cc=catalin.marinas@arm.com \
--cc=daniel@iogearbox.net \
--cc=dvyukov@google.com \
--cc=james.morse@arm.com \
--cc=jean-philippe@linaro.org \
--cc=jpoimboe@redhat.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=maz@kernel.org \
--cc=peterz@infradead.org \
--cc=robin.murphy@arm.com \
--cc=suzuki.poulose@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).