From: Peter Zijlstra <peterz@infradead.org>
To: Markus Trippelsdorf <markus@trippelsdorf.de>
Cc: linux-kernel@vger.kernel.org,
Thomas Gleixner <tglx@linutronix.de>,
Ingo Molnar <mingo@redhat.com>
Subject: Re: Current mainline git (24e700e291d52bd2) hangs when building e.g. perf
Date: Tue, 5 Sep 2017 10:53:50 +0200 [thread overview]
Message-ID: <20170905085350.cgi7shvnillbikow@hirez.programming.kicks-ass.net> (raw)
In-Reply-To: <20170905072738.GA277@x4>
On Tue, Sep 05, 2017 at 09:27:38AM +0200, Markus Trippelsdorf wrote:
> Current mainline git (24e700e291d52bd2) hangs when building software
> concurrently (for example perf).
> The issue is not 100% reproducible (sometimes building perf succeeds),
> so bisecting will not work.
Sadly I cannot reproduce, I had:
while :; do make clean; make; done
running on tools/perf for a while, and now have:
while :; do make O=defconfig-build clean; make O=defconfig-build -j80; done
running, all smooth sailing, although there's the hope that the moment I
hit send on this email the box comes unstuck.
> Magic SysRq key doesn't work and there is nothing in the logs.
> Enabling CONFIG_PROVE_LOCKING makes the issue go away.
SysRq not working is suspicious.. and I take it the NMI watchdog also
isn't firing?
> Any ideas on how to debug this further?
So you have a (real) serial line on that box?
Could you try something like:
debug ignore_loglevel sysrq_always_enabled earlyprintk=serial,ttyS0,115200 force_early_printk
with the below patch applied? That always gives me the most reliable
output.
---
kernel/printk/printk.c | 119 +++++++++++++++++++++++++++++++++++--------------
1 file changed, 86 insertions(+), 33 deletions(-)
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index fc47863f629c..b17099fbc7ce 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -365,6 +365,75 @@ __packed __aligned(4)
#endif
;
+#ifdef CONFIG_EARLY_PRINTK
+struct console *early_console;
+
+static bool __read_mostly force_early_printk;
+
+static int __init force_early_printk_setup(char *str)
+{
+ force_early_printk = true;
+ return 0;
+}
+early_param("force_early_printk", force_early_printk_setup);
+
+static int early_printk_cpu = -1;
+
+static int early_vprintk(const char *fmt, va_list args)
+{
+ int n, cpu, old;
+ char buf[512];
+
+ cpu = get_cpu();
+ /*
+ * Test-and-Set inter-cpu spinlock with recursion.
+ */
+ for (;;) {
+ /*
+ * c-cas to avoid the exclusive bouncing on spin.
+ * Depends on the memory barrier implied by cmpxchg
+ * for ACQUIRE semantics.
+ */
+ old = READ_ONCE(early_printk_cpu);
+ if (old == -1) {
+ old = cmpxchg(&early_printk_cpu, -1, cpu);
+ if (old == -1)
+ break;
+ }
+ /*
+ * Allow recursion for interrupts and the like.
+ */
+ if (old == cpu)
+ break;
+
+ cpu_relax();
+ }
+
+ n = vscnprintf(buf, sizeof(buf), fmt, args);
+ early_console->write(early_console, buf, n);
+
+ /*
+ * Unlock -- in case @old == @cpu, this is a no-op.
+ */
+ smp_store_release(&early_printk_cpu, old);
+ put_cpu();
+
+ return n;
+}
+
+asmlinkage __visible void early_printk(const char *fmt, ...)
+{
+ va_list ap;
+
+ if (!early_console)
+ return;
+
+ va_start(ap, fmt);
+ early_vprintk(fmt, ap);
+ va_end(ap);
+}
+#endif
+
/*
* The logbuf_lock protects kmsg buffer, indices, counters. This can be taken
* within the scheduler's rq lock. It must be released before calling
@@ -1704,6 +1773,16 @@ asmlinkage int vprintk_emit(int facility, int level,
int printed_len = 0;
bool in_sched = false;
+#ifdef CONFIG_KGDB_KDB
+ if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0))
+ return vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args);
+#endif
+
+#ifdef CONFIG_EARLY_PRINTK
+ if (force_early_printk && early_console)
+ return early_vprintk(fmt, args);
+#endif
+
if (level == LOGLEVEL_SCHED) {
level = LOGLEVEL_DEFAULT;
in_sched = true;
@@ -1796,18 +1875,7 @@ EXPORT_SYMBOL(printk_emit);
int vprintk_default(const char *fmt, va_list args)
{
- int r;
-
-#ifdef CONFIG_KGDB_KDB
- /* Allow to pass printk() to kdb but avoid a recursion. */
- if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0)) {
- r = vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args);
- return r;
- }
-#endif
- r = vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args);
-
- return r;
+ return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args);
}
EXPORT_SYMBOL_GPL(vprintk_default);
@@ -1838,7 +1906,12 @@ asmlinkage __visible int printk(const char *fmt, ...)
int r;
va_start(args, fmt);
- r = vprintk_func(fmt, args);
+#ifdef CONFIG_EARLY_PRINTK
+ if (force_early_printk && early_console)
+ r = vprintk_default(fmt, args);
+ else
+#endif
+ r = vprintk_func(fmt, args);
va_end(args);
return r;
@@ -1875,26 +1948,6 @@ static bool suppress_message_printing(int level) { return false; }
#endif /* CONFIG_PRINTK */
-#ifdef CONFIG_EARLY_PRINTK
-struct console *early_console;
-
-asmlinkage __visible void early_printk(const char *fmt, ...)
-{
- va_list ap;
- char buf[512];
- int n;
-
- if (!early_console)
- return;
-
- va_start(ap, fmt);
- n = vscnprintf(buf, sizeof(buf), fmt, ap);
- va_end(ap);
-
- early_console->write(early_console, buf, n);
-}
-#endif
-
static int __add_preferred_console(char *name, int idx, char *options,
char *brl_options)
{
next prev parent reply other threads:[~2017-09-05 8:54 UTC|newest]
Thread overview: 61+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-09-05 7:27 Current mainline git (24e700e291d52bd2) hangs when building e.g. perf Markus Trippelsdorf
2017-09-05 8:53 ` Peter Zijlstra [this message]
2017-09-05 9:55 ` Markus Trippelsdorf
2017-09-06 12:52 ` Thomas Gleixner
2017-09-06 13:15 ` Markus Trippelsdorf
2017-09-07 6:28 ` Markus Trippelsdorf
2017-09-08 5:35 ` Markus Trippelsdorf
2017-09-08 6:26 ` Thomas Gleixner
2017-09-08 8:05 ` Borislav Petkov
2017-09-08 9:16 ` Borislav Petkov
2017-09-08 9:48 ` Markus Trippelsdorf
2017-09-08 10:35 ` Ingo Molnar
2017-09-08 10:39 ` Markus Trippelsdorf
2017-09-08 11:30 ` Markus Trippelsdorf
2017-09-08 16:12 ` Andy Lutomirski
2017-09-08 17:16 ` Markus Trippelsdorf
2017-09-08 21:47 ` Andy Lutomirski
2017-09-08 21:56 ` Borislav Petkov
2017-09-08 23:07 ` Andy Lutomirski
2017-09-08 23:23 ` Linus Torvalds
2017-09-09 0:00 ` Andy Lutomirski
2017-09-09 1:05 ` Linus Torvalds
2017-09-09 1:39 ` Andy Lutomirski
2017-09-09 17:49 ` Andy Lutomirski
2017-09-09 18:02 ` Linus Torvalds
2017-09-09 6:39 ` Markus Trippelsdorf
2017-09-09 10:18 ` Borislav Petkov
2017-09-09 11:07 ` Markus Trippelsdorf
2017-09-09 13:07 ` Borislav Petkov
2017-09-09 13:37 ` Markus Trippelsdorf
2017-09-09 13:39 ` Markus Trippelsdorf
2017-09-09 14:07 ` Borislav Petkov
2017-09-09 14:20 ` Markus Trippelsdorf
2017-09-09 14:33 ` Borislav Petkov
2017-09-09 14:43 ` Markus Trippelsdorf
2017-09-09 16:32 ` Markus Trippelsdorf
2017-09-09 17:05 ` Borislav Petkov
2017-09-09 17:23 ` Markus Trippelsdorf
2017-09-09 17:36 ` Borislav Petkov
2017-09-09 18:14 ` Markus Trippelsdorf
2017-09-09 18:26 ` Borislav Petkov
2017-09-09 18:46 ` Markus Trippelsdorf
2017-09-09 19:11 ` Borislav Petkov
2017-09-09 19:19 ` Borislav Petkov
2017-09-09 18:26 ` Linus Torvalds
2017-09-09 18:29 ` Borislav Petkov
2017-09-09 18:47 ` Linus Torvalds
2017-09-09 19:09 ` Borislav Petkov
2017-09-09 19:21 ` Linus Torvalds
2017-09-09 19:28 ` Andy Lutomirski
2017-09-09 19:37 ` Borislav Petkov
2017-09-10 4:42 ` Andy Lutomirski
2017-09-10 20:22 ` Peter Zijlstra
2017-09-10 20:25 ` Andy Lutomirski
2017-09-17 17:04 ` Ingo Molnar
2017-09-11 1:12 ` Rik van Riel
2017-09-11 1:46 ` Andy Lutomirski
2017-09-11 15:08 ` Rik van Riel
2017-09-12 7:14 ` Markus Trippelsdorf
2017-09-09 8:13 ` Markus Trippelsdorf
2017-09-08 14:51 ` Borislav Petkov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20170905085350.cgi7shvnillbikow@hirez.programming.kicks-ass.net \
--to=peterz@infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=markus@trippelsdorf.de \
--cc=mingo@redhat.com \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).