From: Johan Almbladh <johan.almbladh@anyfinetworks.com>
To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org
Cc: kafai@fb.com, songliubraving@fb.com, yhs@fb.com,
john.fastabend@gmail.com, kpsingh@kernel.org, iii@linux.ibm.com,
paul@cilium.io, yangtiezhu@loongson.cn, netdev@vger.kernel.org,
bpf@vger.kernel.org,
Johan Almbladh <johan.almbladh@anyfinetworks.com>
Subject: [PATCH bpf-next 08/10] bpf/tests: Add exhaustive tests of BPF_ATOMIC register combinations
Date: Fri, 1 Oct 2021 15:03:46 +0200 [thread overview]
Message-ID: <20211001130348.3670534-9-johan.almbladh@anyfinetworks.com> (raw)
In-Reply-To: <20211001130348.3670534-1-johan.almbladh@anyfinetworks.com>
This patch adds tests of all register combinations for BPF_ATOMIC
operations on both BPF_W and BPF_DW sizes.
Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
---
lib/test_bpf.c | 422 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 422 insertions(+)
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 924bf4c9783c..40db4cee4f51 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -1805,6 +1805,246 @@ static int bpf_fill_cmpxchg32(struct bpf_test *self)
&__bpf_emit_cmpxchg32);
}
+/*
+ * Test JITs that implement ATOMIC operations as function calls or
+ * other primitives, and must re-arrange operands for argument passing.
+ */
+static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op)
+{
+ struct bpf_insn *insn;
+ int len = 2 + 34 * 10 * 10;
+ u64 mem, upd, res;
+ int rd, rs, i = 0;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ /* Operand and memory values */
+ if (width == BPF_DW) {
+ mem = 0x0123456789abcdefULL;
+ upd = 0xfedcba9876543210ULL;
+ } else { /* BPF_W */
+ mem = 0x01234567U;
+ upd = 0x76543210U;
+ }
+
+ /* Memory updated according to operation */
+ switch (op) {
+ case BPF_XCHG:
+ res = upd;
+ break;
+ case BPF_CMPXCHG:
+ res = mem;
+ break;
+ default:
+ __bpf_alu_result(&res, mem, upd, BPF_OP(op));
+ }
+
+ /* Test all operand registers */
+ for (rd = R0; rd <= R9; rd++) {
+ for (rs = R0; rs <= R9; rs++) {
+ u64 cmp, src;
+
+ /* Initialize value in memory */
+ i += __bpf_ld_imm64(&insn[i], R0, mem);
+ insn[i++] = BPF_STX_MEM(width, R10, R0, -8);
+
+ /* Initialize registers in order */
+ i += __bpf_ld_imm64(&insn[i], R0, ~mem);
+ i += __bpf_ld_imm64(&insn[i], rs, upd);
+ insn[i++] = BPF_MOV64_REG(rd, R10);
+
+ /* Perform atomic operation */
+ insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8);
+ if (op == BPF_CMPXCHG && width == BPF_W)
+ insn[i++] = BPF_ZEXT_REG(R0);
+
+ /* Check R0 register value */
+ if (op == BPF_CMPXCHG)
+ cmp = mem; /* Expect value from memory */
+ else if (R0 == rd || R0 == rs)
+ cmp = 0; /* Aliased, checked below */
+ else
+ cmp = ~mem; /* Expect value to be preserved */
+ if (cmp) {
+ insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0,
+ (u32)cmp, 2);
+ insn[i++] = BPF_MOV32_IMM(R0, __LINE__);
+ insn[i++] = BPF_EXIT_INSN();
+ insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32);
+ insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0,
+ cmp >> 32, 2);
+ insn[i++] = BPF_MOV32_IMM(R0, __LINE__);
+ insn[i++] = BPF_EXIT_INSN();
+ }
+
+ /* Check source register value */
+ if (rs == R0 && op == BPF_CMPXCHG)
+ src = 0; /* Aliased with R0, checked above */
+ else if (rs == rd && (op == BPF_CMPXCHG ||
+ !(op & BPF_FETCH)))
+ src = 0; /* Aliased with rd, checked below */
+ else if (op == BPF_CMPXCHG)
+ src = upd; /* Expect value to be preserved */
+ else if (op & BPF_FETCH)
+ src = mem; /* Expect fetched value from mem */
+ else /* no fetch */
+ src = upd; /* Expect value to be preserved */
+ if (src) {
+ insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs,
+ (u32)src, 2);
+ insn[i++] = BPF_MOV32_IMM(R0, __LINE__);
+ insn[i++] = BPF_EXIT_INSN();
+ insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32);
+ insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs,
+ src >> 32, 2);
+ insn[i++] = BPF_MOV32_IMM(R0, __LINE__);
+ insn[i++] = BPF_EXIT_INSN();
+ }
+
+ /* Check destination register value */
+ if (!(rd == R0 && op == BPF_CMPXCHG) &&
+ !(rd == rs && (op & BPF_FETCH))) {
+ insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2);
+ insn[i++] = BPF_MOV32_IMM(R0, __LINE__);
+ insn[i++] = BPF_EXIT_INSN();
+ }
+
+ /* Check value in memory */
+ if (rs != rd) { /* No aliasing */
+ i += __bpf_ld_imm64(&insn[i], R1, res);
+ } else if (op == BPF_XCHG) { /* Aliased, XCHG */
+ insn[i++] = BPF_MOV64_REG(R1, R10);
+ } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */
+ i += __bpf_ld_imm64(&insn[i], R1, mem);
+ } else { /* Aliased, ALU oper */
+ i += __bpf_ld_imm64(&insn[i], R1, mem);
+ insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10);
+ }
+
+ insn[i++] = BPF_LDX_MEM(width, R0, R10, -8);
+ if (width == BPF_DW)
+ insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2);
+ else /* width == BPF_W */
+ insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2);
+ insn[i++] = BPF_MOV32_IMM(R0, __LINE__);
+ insn[i++] = BPF_EXIT_INSN();
+ }
+ }
+
+ insn[i++] = BPF_MOV64_IMM(R0, 1);
+ insn[i++] = BPF_EXIT_INSN();
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = i;
+ BUG_ON(i > len);
+
+ return 0;
+}
+
+/* 64-bit atomic register tests */
+static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD);
+}
+
+static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND);
+}
+
+static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR);
+}
+
+static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR);
+}
+
+static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH);
+}
+
+static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH);
+}
+
+static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH);
+}
+
+static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH);
+}
+
+static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG);
+}
+
+static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG);
+}
+
+/* 32-bit atomic register tests */
+static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD);
+}
+
+static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND);
+}
+
+static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR);
+}
+
+static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR);
+}
+
+static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH);
+}
+
+static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH);
+}
+
+static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH);
+}
+
+static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH);
+}
+
+static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG);
+}
+
+static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self)
+{
+ return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG);
+}
+
/*
* Test the two-instruction 64-bit immediate load operation for all
* power-of-two magnitudes of the immediate operand. For each MSB, a block
@@ -11976,6 +12216,188 @@ static struct bpf_test tests[] = {
{ { 0, 1 } },
.fill_helper = bpf_fill_ld_imm64,
},
+ /* 64-bit ATOMIC register combinations */
+ {
+ "ATOMIC_DW_ADD: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_add_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_AND: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_and_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_OR: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_or_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_XOR: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_xor_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_ADD_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_AND_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_OR_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_XOR_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_XCHG: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_xchg_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_DW_CMPXCHG: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs,
+ .stack_depth = 8,
+ },
+ /* 32-bit ATOMIC register combinations */
+ {
+ "ATOMIC_W_ADD: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_add_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_AND: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_and_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_OR: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_or_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_XOR: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_xor_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_ADD_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_AND_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_OR_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_XOR_FETCH: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_XCHG: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_xchg_reg_pairs,
+ .stack_depth = 8,
+ },
+ {
+ "ATOMIC_W_CMPXCHG: register combinations",
+ { },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs,
+ .stack_depth = 8,
+ },
/* 64-bit ATOMIC magnitudes */
{
"ATOMIC_DW_ADD: all operand magnitudes",
--
2.30.2
next prev parent reply other threads:[~2021-10-01 13:05 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-01 13:03 [PATCH bpf-next 00/10] bpf/tests: Extend eBPF JIT test suite Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 01/10] bpf/tests: Add tests of BPF_LDX and BPF_STX with small sizes Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 02/10] bpf/tests: Add zero-extension checks in BPF_ATOMIC tests Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 03/10] bpf/tests: Add exhaustive tests of BPF_ATOMIC magnitudes Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 04/10] bpf/tests: Add tests to check source register zero-extension Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 05/10] bpf/tests: Add more tests for ALU and ATOMIC register clobbering Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 06/10] bpf/tests: Minor restructuring of ALU tests Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 07/10] bpf/tests: Add exhaustive tests of ALU register combinations Johan Almbladh
2021-10-01 13:03 ` Johan Almbladh [this message]
2021-10-01 13:03 ` [PATCH bpf-next 09/10] bpf/tests: Add test of ALU shifts with operand register aliasing Johan Almbladh
2021-10-01 13:03 ` [PATCH bpf-next 10/10] bpf/tests: Add test of LDX_MEM with operand aliasing Johan Almbladh
2021-10-01 15:10 ` [PATCH bpf-next 00/10] bpf/tests: Extend eBPF JIT test suite patchwork-bot+netdevbpf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211001130348.3670534-9-johan.almbladh@anyfinetworks.com \
--to=johan.almbladh@anyfinetworks.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=iii@linux.ibm.com \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@kernel.org \
--cc=netdev@vger.kernel.org \
--cc=paul@cilium.io \
--cc=songliubraving@fb.com \
--cc=yangtiezhu@loongson.cn \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).