[PATCH bpf-next 0/1] selftests/bpf: support custom per-test flags and multiple expected messages

[PATCH bpf-next 0/1] selftests/bpf: support custom per-test flags and multiple expected messages

[Eduard Zingerman]

This patch allows to specify program flags and multiple verifier log
messages for the test_loader kind of tests. For example:

  tools/testing/selftets/bpf/progs/foobar.c:
  
    SEC("tc")
    __success __log_level(7)
    __msg("first message")
    __msg("next message")
    __flag(BPF_F_ANY_ALIGNMENT)
    int buz(struct __sk_buff *skb)
    { ... }

It was developed by Andrii Nakryiko ([1]), I reused it in a
"test_verifier tests migration to inline assembly" patch series ([2]),
but the series is currently stuck on my side.
Andrii asked to spin this particular patch separately ([3]).

[1] https://lore.kernel.org/bpf/CAEf4BzZH0ZxorCi7nPDbRqSK9f+410RooNwNJGwfw8=0a5i1nw@mail.gmail.com/
[2] https://lore.kernel.org/bpf/20230123145148.2791939-1-eddyz87@gmail.com/
[3] https://lore.kernel.org/bpf/20230123145148.2791939-1-eddyz87@gmail.com/T/#m52e806c5a679a2aa8f484d011be7ec105939127a


Andrii Nakryiko (1):
  selftests/bpf: support custom per-test flags and multiple expected
    messages

 tools/testing/selftests/bpf/progs/bpf_misc.h | 23 +++++++
 tools/testing/selftests/bpf/test_loader.c    | 69 +++++++++++++++++---
 tools/testing/selftests/bpf/test_progs.h     |  1 +
 3 files changed, 84 insertions(+), 9 deletions(-)

-- 
2.39.1

[PATCH bpf-next 1/1] selftests/bpf: support custom per-test flags and multiple expected messages

[Eduard Zingerman]

From: Andrii Nakryiko <andrii@kernel.org>

Extend __flag attribute by allowing to specify one of the following:
 * BPF_F_STRICT_ALIGNMENT
 * BPF_F_ANY_ALIGNMENT
 * BPF_F_TEST_RND_HI32
 * BPF_F_TEST_STATE_FREQ
 * BPF_F_SLEEPABLE
 * BPF_F_XDP_HAS_FRAGS
 * Some numeric value

Extend __msg attribute by allowing to specify multiple exepcted messages.
All messages are expected to be present in the verifier log in the
order of application.

Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
[ Eduard: added commit message, formatting, comments ]
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
---
 tools/testing/selftests/bpf/progs/bpf_misc.h | 23 +++++++
 tools/testing/selftests/bpf/test_loader.c    | 69 +++++++++++++++++---
 tools/testing/selftests/bpf/test_progs.h     |  1 +
 3 files changed, 84 insertions(+), 9 deletions(-)

diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h
index 14e28f991451..f704885aa534 100644
--- a/tools/testing/selftests/bpf/progs/bpf_misc.h
+++ b/tools/testing/selftests/bpf/progs/bpf_misc.h
@@ -2,10 +2,33 @@
 #ifndef __BPF_MISC_H__
 #define __BPF_MISC_H__
 
+/* This set of attributes controls behavior of the
+ * test_loader.c:test_loader__run_subtests().
+ *
+ * __msg             Message expected to be found in the verifier log.
+ *                   Multiple __msg attributes could be specified.
+ *
+ * __success         Expect program load success in privileged mode.
+ *
+ * __failure         Expect program load failure in privileged mode.
+ *
+ * __log_level       Log level to use for the program, numeric value expected.
+ *
+ * __flag            Adds one flag use for the program, the following values are valid:
+ *                   - BPF_F_STRICT_ALIGNMENT;
+ *                   - BPF_F_TEST_RND_HI32;
+ *                   - BPF_F_TEST_STATE_FREQ;
+ *                   - BPF_F_SLEEPABLE;
+ *                   - BPF_F_XDP_HAS_FRAGS;
+ *                   - A numeric value.
+ *                   Multiple __flag attributes could be specified, the final flags
+ *                   value is derived by applying binary "or" to all specified values.
+ */
 #define __msg(msg)		__attribute__((btf_decl_tag("comment:test_expect_msg=" msg)))
 #define __failure		__attribute__((btf_decl_tag("comment:test_expect_failure")))
 #define __success		__attribute__((btf_decl_tag("comment:test_expect_success")))
 #define __log_level(lvl)	__attribute__((btf_decl_tag("comment:test_log_level="#lvl)))
+#define __flag(flag)		__attribute__((btf_decl_tag("comment:test_prog_flags="#flag)))
 
 /* Convenience macro for use with 'asm volatile' blocks */
 #define __naked __attribute__((naked))
diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c
index 679efb3aa785..bf41390157bf 100644
--- a/tools/testing/selftests/bpf/test_loader.c
+++ b/tools/testing/selftests/bpf/test_loader.c
@@ -13,12 +13,15 @@
 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"
 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="
 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="
+#define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="
 
 struct test_spec {
 	const char *name;
 	bool expect_failure;
-	const char *expect_msg;
+	const char **expect_msgs;
+	size_t expect_msg_cnt;
 	int log_level;
+	int prog_flags;
 };
 
 static int tester_init(struct test_loader *tester)
@@ -67,7 +70,8 @@ static int parse_test_spec(struct test_loader *tester,
 
 	for (i = 1; i < btf__type_cnt(btf); i++) {
 		const struct btf_type *t;
-		const char *s;
+		const char *s, *val;
+		char *e;
 
 		t = btf__type_by_id(btf, i);
 		if (!btf_is_decl_tag(t))
@@ -82,14 +86,48 @@ static int parse_test_spec(struct test_loader *tester,
 		} else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {
 			spec->expect_failure = false;
 		} else if (str_has_pfx(s, TEST_TAG_EXPECT_MSG_PFX)) {
-			spec->expect_msg = s + sizeof(TEST_TAG_EXPECT_MSG_PFX) - 1;
+			void *tmp;
+			const char **msg;
+
+			tmp = realloc(spec->expect_msgs,
+				      (1 + spec->expect_msg_cnt) * sizeof(void *));
+			if (!tmp) {
+				ASSERT_FAIL("failed to realloc memory for messages\n");
+				return -ENOMEM;
+			}
+			spec->expect_msgs = tmp;
+			msg = &spec->expect_msgs[spec->expect_msg_cnt++];
+			*msg = s + sizeof(TEST_TAG_EXPECT_MSG_PFX) - 1;
 		} else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {
+			val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;
 			errno = 0;
-			spec->log_level = strtol(s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1, NULL, 0);
-			if (errno) {
+			spec->log_level = strtol(val, &e, 0);
+			if (errno || e[0] != '\0') {
 				ASSERT_FAIL("failed to parse test log level from '%s'", s);
 				return -EINVAL;
 			}
+		} else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
+			val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
+			if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
+				spec->prog_flags |= BPF_F_STRICT_ALIGNMENT;
+			} else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
+				spec->prog_flags |= BPF_F_ANY_ALIGNMENT;
+			} else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
+				spec->prog_flags |= BPF_F_TEST_RND_HI32;
+			} else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
+				spec->prog_flags |= BPF_F_TEST_STATE_FREQ;
+			} else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
+				spec->prog_flags |= BPF_F_SLEEPABLE;
+			} else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
+				spec->prog_flags |= BPF_F_XDP_HAS_FRAGS;
+			} else /* assume numeric value */ {
+				errno = 0;
+				spec->prog_flags |= strtol(val, &e, 0);
+				if (errno || e[0] != '\0') {
+					ASSERT_FAIL("failed to parse test prog flags from '%s'", s);
+					return -EINVAL;
+				}
+			}
 		}
 	}
 
@@ -101,7 +139,7 @@ static void prepare_case(struct test_loader *tester,
 			 struct bpf_object *obj,
 			 struct bpf_program *prog)
 {
-	int min_log_level = 0;
+	int min_log_level = 0, prog_flags;
 
 	if (env.verbosity > VERBOSE_NONE)
 		min_log_level = 1;
@@ -119,7 +157,11 @@ static void prepare_case(struct test_loader *tester,
 	else
 		bpf_program__set_log_level(prog, spec->log_level);
 
+	prog_flags = bpf_program__flags(prog);
+	bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
+
 	tester->log_buf[0] = '\0';
+	tester->next_match_pos = 0;
 }
 
 static void emit_verifier_log(const char *log_buf, bool force)
@@ -135,17 +177,26 @@ static void validate_case(struct test_loader *tester,
 			  struct bpf_program *prog,
 			  int load_err)
 {
-	if (spec->expect_msg) {
+	int i, j;
+
+	for (i = 0; i < spec->expect_msg_cnt; i++) {
 		char *match;
+		const char *expect_msg;
+
+		expect_msg = spec->expect_msgs[i];
 
-		match = strstr(tester->log_buf, spec->expect_msg);
+		match = strstr(tester->log_buf + tester->next_match_pos, expect_msg);
 		if (!ASSERT_OK_PTR(match, "expect_msg")) {
 			/* if we are in verbose mode, we've already emitted log */
 			if (env.verbosity == VERBOSE_NONE)
 				emit_verifier_log(tester->log_buf, true /*force*/);
-			fprintf(stderr, "EXPECTED MSG: '%s'\n", spec->expect_msg);
+			for (j = 0; j < i; j++)
+				fprintf(stderr, "MATCHED  MSG: '%s'\n", spec->expect_msgs[j]);
+			fprintf(stderr, "EXPECTED MSG: '%s'\n", expect_msg);
 			return;
 		}
+
+		tester->next_match_pos = match - tester->log_buf + strlen(expect_msg);
 	}
 }
 
diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h
index 9fbdc57c5b57..3cbf005747ed 100644
--- a/tools/testing/selftests/bpf/test_progs.h
+++ b/tools/testing/selftests/bpf/test_progs.h
@@ -427,6 +427,7 @@ int get_bpf_max_tramp_links(void);
 struct test_loader {
 	char *log_buf;
 	size_t log_buf_sz;
+	size_t next_match_pos;
 
 	struct bpf_object *obj;
 };
-- 
2.39.1

Re: [PATCH bpf-next 0/1] selftests/bpf: support custom per-test flags and multiple expected messages

[patchwork-bot+netdevbpf]

Hello:

This patch was applied to bpf/bpf-next.git (master)
by Andrii Nakryiko <andrii@kernel.org>:

On Wed,  1 Mar 2023 19:54:16 +0200 you wrote:
> This patch allows to specify program flags and multiple verifier log
> messages for the test_loader kind of tests. For example:
> 
>   tools/testing/selftets/bpf/progs/foobar.c:
> 
>     SEC("tc")
>     __success __log_level(7)
>     __msg("first message")
>     __msg("next message")
>     __flag(BPF_F_ANY_ALIGNMENT)
>     int buz(struct __sk_buff *skb)
>     { ... }
> 
> [...]

Here is the summary with links:
  - [bpf-next,1/1] selftests/bpf: support custom per-test flags and multiple expected messages
    https://git.kernel.org/bpf/bpf-next/c/35cbf7f91568

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html