[PATCH 0/2] Skip the 'perf test sigtrap' on kernels with sleepable spinlocks

[PATCH 0/2] Skip the 'perf test sigtrap' on kernels with sleepable spinlocks

[Arnaldo Carvalho de Melo]

From: Arnaldo Carvalho de Melo <acme@redhat.com>

Hi,

	Please take a look,

- Arnaldo

Arnaldo Carvalho de Melo (2):
  perf test sigtrap: Generalize the BTF routine to reuse it in this test
  perf tests sigtrap: Skip if running on a kernel with sleepable
    spinlocks

 tools/perf/tests/sigtrap.c | 106 +++++++++++++++++++++++++++++--------
 1 file changed, 84 insertions(+), 22 deletions(-)

-- 
2.41.0

[PATCH 1/2] perf test sigtrap: Generalize the BTF routine to reuse it in this test

[Arnaldo Carvalho de Melo]

From: Arnaldo Carvalho de Melo <acme@redhat.com>

Move the part that loads the BTF info to a "btf__available()" that will
lazy load the BTF info so that if we need it for some other test, which
we will in the following cset, we can reuse it.

At some point this will move from this specific 'perf test' entry to be
used in other parts of perf, do it when needed.

Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: Ian Rogers <irogers@google.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/tests/sigtrap.c | 60 +++++++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/tools/perf/tests/sigtrap.c b/tools/perf/tests/sigtrap.c
index 1de7478ec1894d77..a1bc7c776254ed2f 100644
--- a/tools/perf/tests/sigtrap.c
+++ b/tools/perf/tests/sigtrap.c
@@ -57,36 +57,51 @@ static struct perf_event_attr make_event_attr(void)
 #ifdef HAVE_BPF_SKEL
 #include <bpf/btf.h>
 
-static bool attr_has_sigtrap(void)
+static struct btf *btf;
+
+static bool btf__available(void)
 {
-	bool ret = false;
-	struct btf *btf;
-	const struct btf_type *t;
+	if (btf == NULL)
+		btf = btf__load_vmlinux_btf();
+
+	return btf != NULL;
+}
+
+static void btf__exit(void)
+{
+	btf__free(btf);
+	btf = NULL;
+}
+
+static const struct btf_member *__btf_type__find_member_by_name(int type_id, const char *member_name)
+{
+	const struct btf_type *t = btf__type_by_id(btf, type_id);
 	const struct btf_member *m;
-	const char *name;
-	int i, id;
+	int i;
+
+	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
+		const char *current_member_name = btf__name_by_offset(btf, m->name_off);
+		if (!strcmp(current_member_name, member_name))
+			return m;
+	}
+
+	return NULL;
+}
+
+static bool attr_has_sigtrap(void)
+{
+	int id;
 
-	btf = btf__load_vmlinux_btf();
-	if (btf == NULL) {
+	if (!btf__available()) {
 		/* should be an old kernel */
 		return false;
 	}
 
 	id = btf__find_by_name_kind(btf, "perf_event_attr", BTF_KIND_STRUCT);
 	if (id < 0)
-		goto out;
+		return false;
 
-	t = btf__type_by_id(btf, id);
-	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
-		name = btf__name_by_offset(btf, m->name_off);
-		if (!strcmp(name, "sigtrap")) {
-			ret = true;
-			break;
-		}
-	}
-out:
-	btf__free(btf);
-	return ret;
+	return __btf_type__find_member_by_name(id, "sigtrap") != NULL;
 }
 #else  /* !HAVE_BPF_SKEL */
 static bool attr_has_sigtrap(void)
@@ -109,6 +124,10 @@ static bool attr_has_sigtrap(void)
 
 	return ret;
 }
+
+static void btf__exit(void)
+{
+}
 #endif  /* HAVE_BPF_SKEL */
 
 static void
@@ -221,6 +240,7 @@ static int test__sigtrap(struct test_suite *test __maybe_unused, int subtest __m
 	sigaction(SIGTRAP, &oldact, NULL);
 out:
 	pthread_barrier_destroy(&barrier);
+	btf__exit();
 	return ret;
 }
 
-- 
2.41.0

[PATCH 2/2] perf tests sigtrap: Skip if running on a kernel with sleepable spinlocks

[Arnaldo Carvalho de Melo]

From: Arnaldo Carvalho de Melo <acme@redhat.com>

There are issues as reported that need some more investigation on the
RT kernel front, till that is addressed, skip this test.

This test is already skipped for multiple hardware architectures where
the tested kernel feature is not supported.

Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Ian Rogers <irogers@google.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Marco Elver <elver@google.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/tests/sigtrap.c | 46 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/tools/perf/tests/sigtrap.c b/tools/perf/tests/sigtrap.c
index a1bc7c776254ed2f..e6fd934b027a3d0c 100644
--- a/tools/perf/tests/sigtrap.c
+++ b/tools/perf/tests/sigtrap.c
@@ -103,6 +103,34 @@ static bool attr_has_sigtrap(void)
 
 	return __btf_type__find_member_by_name(id, "sigtrap") != NULL;
 }
+
+static bool kernel_with_sleepable_spinlocks(void)
+{
+	const struct btf_member *member;
+	const struct btf_type *type;
+	const char *type_name;
+	int id;
+
+	if (!btf__available())
+		return false;
+
+	id = btf__find_by_name_kind(btf, "spinlock", BTF_KIND_STRUCT);
+	if (id < 0)
+		return false;
+
+	// Only RT has a "lock" member for "struct spinlock"
+	member = __btf_type__find_member_by_name(id, "lock");
+	if (member == NULL)
+		return false;
+
+	// But check its type as well
+	type = btf__type_by_id(btf, member->type);
+	if (!type || !btf_is_struct(type))
+		return false;
+
+	type_name = btf__name_by_offset(btf, type->name_off);
+	return type_name && !strcmp(type_name, "rt_mutex_base");
+}
 #else  /* !HAVE_BPF_SKEL */
 static bool attr_has_sigtrap(void)
 {
@@ -125,6 +153,11 @@ static bool attr_has_sigtrap(void)
 	return ret;
 }
 
+static bool kernel_with_sleepable_spinlocks(void)
+{
+	return false;
+}
+
 static void btf__exit(void)
 {
 }
@@ -166,7 +199,7 @@ static int run_test_threads(pthread_t *threads, pthread_barrier_t *barrier)
 
 static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrier)
 {
-	int ret;
+	int ret, expected_sigtraps;
 
 	ctx.iterate_on = 3000;
 
@@ -175,7 +208,16 @@ static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrie
 	ret = run_test_threads(threads, barrier);
 	TEST_ASSERT_EQUAL("disable failed", ioctl(fd, PERF_EVENT_IOC_DISABLE, 0), 0);
 
-	TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, NUM_THREADS * ctx.iterate_on);
+	expected_sigtraps = NUM_THREADS * ctx.iterate_on;
+
+	if (ctx.signal_count < expected_sigtraps && kernel_with_sleepable_spinlocks()) {
+		pr_debug("Expected %d sigtraps, got %d, running on a kernel with sleepable spinlocks.\n",
+			 expected_sigtraps, ctx.signal_count);
+		pr_debug("See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/\n");
+		return TEST_SKIP;
+	} else
+		TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, expected_sigtraps);
+
 	TEST_ASSERT_EQUAL("missing signals or incorrectly delivered", ctx.tids_want_signal, 0);
 	TEST_ASSERT_VAL("unexpected si_addr", ctx.first_siginfo.si_addr == &ctx.iterate_on);
 #if 0 /* FIXME: enable when libc's signal.h has si_perf_{type,data} */
-- 
2.41.0

Re: [PATCH 2/2] perf tests sigtrap: Skip if running on a kernel with sleepable spinlocks

[Marco Elver]

On Wed, 29 Nov 2023 at 16:47, Arnaldo Carvalho de Melo <acme@kernel.org> wrote:
>
> From: Arnaldo Carvalho de Melo <acme@redhat.com>
>
> There are issues as reported that need some more investigation on the
> RT kernel front, till that is addressed, skip this test.
>
> This test is already skipped for multiple hardware architectures where
> the tested kernel feature is not supported.
>
> Cc: Adrian Hunter <adrian.hunter@intel.com>
> Cc: Clark Williams <williams@redhat.com>
> Cc: Ian Rogers <irogers@google.com>
> Cc: Jiri Olsa <jolsa@kernel.org>
> Cc: Juri Lelli <juri.lelli@redhat.com>
> Cc: Marco Elver <elver@google.com>
> Cc: Mike Galbraith <efault@gmx.de>
> Cc: Namhyung Kim <namhyung@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Link: https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/
> Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>

Acked-by: Marco Elver <elver@google.com>

> ---
>  tools/perf/tests/sigtrap.c | 46 ++++++++++++++++++++++++++++++++++++--
>  1 file changed, 44 insertions(+), 2 deletions(-)
>
> diff --git a/tools/perf/tests/sigtrap.c b/tools/perf/tests/sigtrap.c
> index a1bc7c776254ed2f..e6fd934b027a3d0c 100644
> --- a/tools/perf/tests/sigtrap.c
> +++ b/tools/perf/tests/sigtrap.c
> @@ -103,6 +103,34 @@ static bool attr_has_sigtrap(void)
>
>         return __btf_type__find_member_by_name(id, "sigtrap") != NULL;
>  }
> +
> +static bool kernel_with_sleepable_spinlocks(void)
> +{
> +       const struct btf_member *member;
> +       const struct btf_type *type;
> +       const char *type_name;
> +       int id;
> +
> +       if (!btf__available())
> +               return false;
> +
> +       id = btf__find_by_name_kind(btf, "spinlock", BTF_KIND_STRUCT);
> +       if (id < 0)
> +               return false;
> +
> +       // Only RT has a "lock" member for "struct spinlock"
> +       member = __btf_type__find_member_by_name(id, "lock");
> +       if (member == NULL)
> +               return false;
> +
> +       // But check its type as well
> +       type = btf__type_by_id(btf, member->type);
> +       if (!type || !btf_is_struct(type))
> +               return false;
> +
> +       type_name = btf__name_by_offset(btf, type->name_off);
> +       return type_name && !strcmp(type_name, "rt_mutex_base");
> +}
>  #else  /* !HAVE_BPF_SKEL */
>  static bool attr_has_sigtrap(void)
>  {
> @@ -125,6 +153,11 @@ static bool attr_has_sigtrap(void)
>         return ret;
>  }
>
> +static bool kernel_with_sleepable_spinlocks(void)
> +{
> +       return false;
> +}
> +
>  static void btf__exit(void)
>  {
>  }
> @@ -166,7 +199,7 @@ static int run_test_threads(pthread_t *threads, pthread_barrier_t *barrier)
>
>  static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrier)
>  {
> -       int ret;
> +       int ret, expected_sigtraps;
>
>         ctx.iterate_on = 3000;
>
> @@ -175,7 +208,16 @@ static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrie
>         ret = run_test_threads(threads, barrier);
>         TEST_ASSERT_EQUAL("disable failed", ioctl(fd, PERF_EVENT_IOC_DISABLE, 0), 0);
>
> -       TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, NUM_THREADS * ctx.iterate_on);
> +       expected_sigtraps = NUM_THREADS * ctx.iterate_on;
> +
> +       if (ctx.signal_count < expected_sigtraps && kernel_with_sleepable_spinlocks()) {
> +               pr_debug("Expected %d sigtraps, got %d, running on a kernel with sleepable spinlocks.\n",
> +                        expected_sigtraps, ctx.signal_count);
> +               pr_debug("See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/\n");

No changes from the RT side since? A fix exists, but apparently not
good enough... Sigh.

> +               return TEST_SKIP;
> +       } else
> +               TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, expected_sigtraps);
> +
>         TEST_ASSERT_EQUAL("missing signals or incorrectly delivered", ctx.tids_want_signal, 0);
>         TEST_ASSERT_VAL("unexpected si_addr", ctx.first_siginfo.si_addr == &ctx.iterate_on);
>  #if 0 /* FIXME: enable when libc's signal.h has si_perf_{type,data} */
> --
> 2.41.0
>

Re: [PATCH 2/2] perf tests sigtrap: Skip if running on a kernel with sleepable spinlocks

[Arnaldo Carvalho de Melo]

Em Wed, Nov 29, 2023 at 04:57:47PM +0100, Marco Elver escreveu:
> On Wed, 29 Nov 2023 at 16:47, Arnaldo Carvalho de Melo <acme@kernel.org> wrote:
> >
> > From: Arnaldo Carvalho de Melo <acme@redhat.com>
> >
> > There are issues as reported that need some more investigation on the
> > RT kernel front, till that is addressed, skip this test.
> >
> > This test is already skipped for multiple hardware architectures where
> > the tested kernel feature is not supported.
> >
> > Cc: Adrian Hunter <adrian.hunter@intel.com>
> > Cc: Clark Williams <williams@redhat.com>
> > Cc: Ian Rogers <irogers@google.com>
> > Cc: Jiri Olsa <jolsa@kernel.org>
> > Cc: Juri Lelli <juri.lelli@redhat.com>
> > Cc: Marco Elver <elver@google.com>
> > Cc: Mike Galbraith <efault@gmx.de>
> > Cc: Namhyung Kim <namhyung@kernel.org>
> > Cc: Peter Zijlstra <peterz@infradead.org>
> > Cc: Thomas Gleixner <tglx@linutronix.de>
> > Link: https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/
> > Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
> 
> Acked-by: Marco Elver <elver@google.com>
> 
> > ---
> >  tools/perf/tests/sigtrap.c | 46 ++++++++++++++++++++++++++++++++++++--
> >  1 file changed, 44 insertions(+), 2 deletions(-)
> >
> > diff --git a/tools/perf/tests/sigtrap.c b/tools/perf/tests/sigtrap.c
> > index a1bc7c776254ed2f..e6fd934b027a3d0c 100644
> > --- a/tools/perf/tests/sigtrap.c
> > +++ b/tools/perf/tests/sigtrap.c
> > @@ -103,6 +103,34 @@ static bool attr_has_sigtrap(void)
> >
> >         return __btf_type__find_member_by_name(id, "sigtrap") != NULL;
> >  }
> > +
> > +static bool kernel_with_sleepable_spinlocks(void)
> > +{
> > +       const struct btf_member *member;
> > +       const struct btf_type *type;
> > +       const char *type_name;
> > +       int id;
> > +
> > +       if (!btf__available())
> > +               return false;
> > +
> > +       id = btf__find_by_name_kind(btf, "spinlock", BTF_KIND_STRUCT);
> > +       if (id < 0)
> > +               return false;
> > +
> > +       // Only RT has a "lock" member for "struct spinlock"
> > +       member = __btf_type__find_member_by_name(id, "lock");
> > +       if (member == NULL)
> > +               return false;
> > +
> > +       // But check its type as well
> > +       type = btf__type_by_id(btf, member->type);
> > +       if (!type || !btf_is_struct(type))
> > +               return false;
> > +
> > +       type_name = btf__name_by_offset(btf, type->name_off);
> > +       return type_name && !strcmp(type_name, "rt_mutex_base");
> > +}
> >  #else  /* !HAVE_BPF_SKEL */
> >  static bool attr_has_sigtrap(void)
> >  {
> > @@ -125,6 +153,11 @@ static bool attr_has_sigtrap(void)
> >         return ret;
> >  }
> >
> > +static bool kernel_with_sleepable_spinlocks(void)
> > +{
> > +       return false;
> > +}
> > +
> >  static void btf__exit(void)
> >  {
> >  }
> > @@ -166,7 +199,7 @@ static int run_test_threads(pthread_t *threads, pthread_barrier_t *barrier)
> >
> >  static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrier)
> >  {
> > -       int ret;
> > +       int ret, expected_sigtraps;
> >
> >         ctx.iterate_on = 3000;
> >
> > @@ -175,7 +208,16 @@ static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrie
> >         ret = run_test_threads(threads, barrier);
> >         TEST_ASSERT_EQUAL("disable failed", ioctl(fd, PERF_EVENT_IOC_DISABLE, 0), 0);
> >
> > -       TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, NUM_THREADS * ctx.iterate_on);
> > +       expected_sigtraps = NUM_THREADS * ctx.iterate_on;
> > +
> > +       if (ctx.signal_count < expected_sigtraps && kernel_with_sleepable_spinlocks()) {
> > +               pr_debug("Expected %d sigtraps, got %d, running on a kernel with sleepable spinlocks.\n",
> > +                        expected_sigtraps, ctx.signal_count);
> > +               pr_debug("See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/\n");
> 
> No changes from the RT side since? A fix exists, but apparently not
> good enough... Sigh.

Yeah, my impression, and first attempt at writing that patch wast that
no sigtraps were being sent, but then when I tried with a random, more
recent machine in the Red Hat labs, I got some signals, way less than
the expected ones, but some, maybe this is an interesting data point?

I'll try again to reproduce in the local machine, old i7 lenovo notebook
and at the newer machine, a Xeon(R) Silver 4216, 32 cpu and report here.

- Arnaldo

Re: [PATCH 2/2] perf tests sigtrap: Skip if running on a kernel with sleepable spinlocks

[Arnaldo Carvalho de Melo]

Em Wed, Nov 29, 2023 at 05:42:30PM -0300, Arnaldo Carvalho de Melo escreveu:
> Em Wed, Nov 29, 2023 at 04:57:47PM +0100, Marco Elver escreveu:
> > > @@ -175,7 +208,16 @@ static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrie
> > >         ret = run_test_threads(threads, barrier);
> > >         TEST_ASSERT_EQUAL("disable failed", ioctl(fd, PERF_EVENT_IOC_DISABLE, 0), 0);

> > > -       TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, NUM_THREADS * ctx.iterate_on);
> > > +       expected_sigtraps = NUM_THREADS * ctx.iterate_on;

> > > +       if (ctx.signal_count < expected_sigtraps && kernel_with_sleepable_spinlocks()) {
> > > +               pr_debug("Expected %d sigtraps, got %d, running on a kernel with sleepable spinlocks.\n",
> > > +                        expected_sigtraps, ctx.signal_count);
> > > +               pr_debug("See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/\n");

> > No changes from the RT side since? A fix exists, but apparently not
> > good enough... Sigh.

> Yeah, my impression, and first attempt at writing that patch wast that
> no sigtraps were being sent, but then when I tried with a random, more
> recent machine in the Red Hat labs, I got some signals, way less than
> the expected ones, but some, maybe this is an interesting data point?
 
> I'll try again to reproduce in the local machine, old i7 lenovo notebook
> and at the newer machine, a Xeon(R) Silver 4216, 32 cpu and report here.

So, on the i7 lenovo:

[root@nine ~]# uname -a
Linux nine 5.14.0-284.30.1.rt14.315.el9_2.x86_64 #1 SMP PREEMPT_RT Fri Aug 25 10:53:59 EDT 2023 x86_64 x86_64 x86_64 GNU/Linux
[root@nine ~]# grep "model name" /proc/cpuinfo
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
model name	: Intel(R) Core(TM) i7-2920XM CPU @ 2.50GHz
[root@nine ~]# grep "model name" /proc/cpuinfo  | wc -l
8
[root@nine ~]#
[root@nine ~]# perf test -v sigtrap
 68: Sigtrap                                                         :
--- start ---
test child forked, pid 77679
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/
test child finished with -2
---- end ----
Sigtrap: Skip
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# perf test -v sigtrap |& grep Expected
Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# uname -a

Consistently 0 sigtraps delivered:

[root@nine ~]# for a in $(seq 100) ; do perf test -v sigtrap |& grep Expected ; done | sort | uniq -c
    100 Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]# for a in $(seq 1000) ; do perf test -v sigtrap |& grep Expected ; done | sort | uniq -c
   1000 Expected 15000 sigtraps, got 0, running on a kernel with sleepable spinlocks.
[root@nine ~]#

While on the bigger machine:

[root@perf160 ~]# uname -a
Linux perf160.perf.lab.eng.bos.redhat.com 5.14.0-362.8.1.el9_3.x86_64+rt #1 SMP PREEMPT_RT Tue Oct 3 10:26:54 EDT 2023 x86_64 x86_64 x86_64 GNU/Linux
[root@perf160 ~]#

[acme@perf160 ~]$ grep "model name" /proc/cpuinfo  | wc -l
32
[acme@perf160 ~]$ grep "model name" /proc/cpuinfo | head -1
model name	: Intel(R) Xeon(R) Silver 4216 CPU @ 2.10GHz
[acme@perf160 ~]$
[acme@perf160 ~]$ perf test -v sigtrap
 68: Sigtrap                                                         :
--- start ---
test child forked, pid 72084
Expected 15000 sigtraps, got 1845, running on a kernel with sleepable spinlocks.
See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/
test child finished with -2
---- end ----
Sigtrap: Skip
[acme@perf160 ~]$ perf test -v sigtrap
 68: Sigtrap                                                         :
--- start ---
test child forked, pid 72091
Expected 15000 sigtraps, got 2060, running on a kernel with sleepable spinlocks.
See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/
test child finished with -2
---- end ----
Sigtrap: Skip
[acme@perf160 ~]$

[root@perf160 ~]# for a in $(seq 100) ; do perf test -v sigtrap |& grep Expected ; done | sort | uniq -c | sort -n
      1 Expected 15000 sigtraps, got 1010, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1064, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1139, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1165, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1166, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1177, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1206, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1279, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1321, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1359, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1368, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1400, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1432, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1490, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1520, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1527, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1532, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1566, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1597, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1600, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1630, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1652, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1689, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1706, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1709, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1753, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1765, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1778, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1830, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1896, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1901, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1903, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1908, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1909, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1930, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1951, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1976, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 1980, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2010, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2012, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2071, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2075, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2166, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2169, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2185, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2189, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2229, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2241, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2249, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2297, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2303, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2313, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2325, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2326, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2350, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2359, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2378, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2448, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2479, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2480, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2489, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2501, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2569, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2573, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2597, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2605, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2639, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2647, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2719, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2754, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2804, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2805, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2860, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 2882, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3152, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3177, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3179, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3249, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3261, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3332, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3388, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3395, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3465, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3487, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3622, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3677, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3782, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 3901, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 4087, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 4235, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 4372, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 4570, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 571, running on a kernel with sleepable spinlocks.
      1 Expected 15000 sigtraps, got 622, running on a kernel with sleepable spinlocks.
      2 Expected 15000 sigtraps, got 1929, running on a kernel with sleepable spinlocks.
      2 Expected 15000 sigtraps, got 1967, running on a kernel with sleepable spinlocks.
      2 Expected 15000 sigtraps, got 2072, running on a kernel with sleepable spinlocks.
[root@perf160 ~]# 

I guess I'll try to get hold of the older kernel with 0 sigtraps to see
if I get the same behaviour (consistent 0 sigtraps) on that kernel on
the bigger machine :-\

- Arnaldo

Re: [PATCH 2/2] perf tests sigtrap: Skip if running on a kernel with sleepable spinlocks

[Marco Elver]

On Thu, 30 Nov 2023 at 14:01, Arnaldo Carvalho de Melo <acme@kernel.org> wrote:
>
> Em Wed, Nov 29, 2023 at 05:42:30PM -0300, Arnaldo Carvalho de Melo escreveu:
> > Em Wed, Nov 29, 2023 at 04:57:47PM +0100, Marco Elver escreveu:
> > > > @@ -175,7 +208,16 @@ static int run_stress_test(int fd, pthread_t *threads, pthread_barrier_t *barrie
> > > >         ret = run_test_threads(threads, barrier);
> > > >         TEST_ASSERT_EQUAL("disable failed", ioctl(fd, PERF_EVENT_IOC_DISABLE, 0), 0);
>
> > > > -       TEST_ASSERT_EQUAL("unexpected sigtraps", ctx.signal_count, NUM_THREADS * ctx.iterate_on);
> > > > +       expected_sigtraps = NUM_THREADS * ctx.iterate_on;
>
> > > > +       if (ctx.signal_count < expected_sigtraps && kernel_with_sleepable_spinlocks()) {
> > > > +               pr_debug("Expected %d sigtraps, got %d, running on a kernel with sleepable spinlocks.\n",
> > > > +                        expected_sigtraps, ctx.signal_count);
> > > > +               pr_debug("See https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/\n");
>
> > > No changes from the RT side since? A fix exists, but apparently not
> > > good enough... Sigh.
>
> > Yeah, my impression, and first attempt at writing that patch wast that
> > no sigtraps were being sent, but then when I tried with a random, more
> > recent machine in the Red Hat labs, I got some signals, way less than
> > the expected ones, but some, maybe this is an interesting data point?
>
> > I'll try again to reproduce in the local machine, old i7 lenovo notebook
> > and at the newer machine, a Xeon(R) Silver 4216, 32 cpu and report here.
>
> So, on the i7 lenovo:
>
> [root@nine ~]# uname -a
> Linux nine 5.14.0-284.30.1.rt14.315.el9_2.x86_64 #1 SMP PREEMPT_RT Fri Aug 25 10:53:59 EDT 2023 x86_64 x86_64 x86_64 GNU/Linux
[...]
>
> I guess I'll try to get hold of the older kernel with 0 sigtraps to see
> if I get the same behaviour (consistent 0 sigtraps) on that kernel on
> the bigger machine :-\

Thanks for checking.

In any case, it looks like it's still broken. If the fix (bf9ad37dc8a
+ small diff by Mike) from [1] still works, what's blocking it from
being upstreamed?

https://lore.kernel.org/all/e368f2c848d77fbc8d259f44e2055fe469c219cf.camel@gmx.de/