* [PATCH v2] x86: show remote CPU state upon fatal NMI or unknown MCE
@ 2016-06-16 11:26 Jan Beulich
2016-06-16 11:32 ` Andrew Cooper
0 siblings, 1 reply; 2+ messages in thread
From: Jan Beulich @ 2016-06-16 11:26 UTC (permalink / raw)
To: xen-devel; +Cc: Andrew Cooper
[-- Attachment #1: Type: text/plain, Size: 6034 bytes --]
Quite frequently the watchdog would hit an innocent CPU, e.g. one
trying to acquire a spin lock a remote CPU holds for extended periods
of time, or a random CPU in TSC calbration rendezvous. In such cases
the register and stack dump for that CPU doesn't really help in the
analysis of the problem.
To keep things reasonable on large systems, only log CS:RIP by default.
This can be overridden via a new command line option such that full
register/stack state would get logged.
Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v2: Pass flag to fatal_trap().
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -198,6 +198,14 @@ Permit Xen to use Address Space Identifi
tags the TLB entries with an ID per vcpu. This allows for guest TLB flushes
to be performed without the overhead of a complete TLB flush.
+### async-show-all
+> `= <boolean>`
+
+> Default: `false`
+
+Forces all CPUs' full state to be logged upon certain fatal asynchronous
+exceptions (watchdog NMIs and unexpected MCEs).
+
### ats
> `= <boolean>`
--- a/xen/arch/x86/cpu/mcheck/mce.c
+++ b/xen/arch/x86/cpu/mcheck/mce.c
@@ -77,7 +77,7 @@ static void unexpected_machine_check(con
{
console_force_unlock();
printk("Unexpected Machine Check Exception\n");
- fatal_trap(regs);
+ fatal_trap(regs, 1);
}
--- a/xen/arch/x86/nmi.c
+++ b/xen/arch/x86/nmi.c
@@ -488,7 +488,7 @@ bool_t nmi_watchdog_tick(const struct cp
console_force_unlock();
printk("Watchdog timer detects that CPU%d is stuck!\n",
smp_processor_id());
- fatal_trap(regs);
+ fatal_trap(regs, 1);
}
}
else
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -525,6 +525,25 @@ void vcpu_show_execution_state(struct vc
vcpu_unpause(v);
}
+static cpumask_t show_state_mask;
+static bool_t opt_show_all;
+boolean_param("async-show-all", opt_show_all);
+
+static int nmi_show_execution_state(const struct cpu_user_regs *regs, int cpu)
+{
+ if ( !cpumask_test_cpu(cpu, &show_state_mask) )
+ return 0;
+
+ if ( opt_show_all )
+ show_execution_state(regs);
+ else
+ printk(XENLOG_ERR "CPU%d @ %04x:%08lx (%pS)\n", cpu, regs->cs, regs->rip,
+ guest_mode(regs) ? _p(regs->rip) : NULL);
+ cpumask_clear_cpu(cpu, &show_state_mask);
+
+ return 1;
+}
+
static const char *trapstr(unsigned int trapnr)
{
static const char * const strings[] = {
@@ -544,7 +563,7 @@ static const char *trapstr(unsigned int
* are disabled). In such situations we can't do much that is safe. We try to
* print out some tracing and then we just spin.
*/
-void fatal_trap(const struct cpu_user_regs *regs)
+void fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote)
{
static DEFINE_PER_CPU(char, depth);
unsigned int trapnr = regs->entry_vector;
@@ -570,6 +589,15 @@ void fatal_trap(const struct cpu_user_re
printk("Faulting linear address: %p\n", _p(cr2));
show_page_walk(cr2);
}
+ if ( show_remote )
+ {
+ cpumask_andnot(&show_state_mask, &cpu_online_map,
+ cpumask_of(smp_processor_id()));
+ set_nmi_callback(nmi_show_execution_state);
+ smp_send_nmi_allbutself();
+ while ( !cpumask_empty(&show_state_mask) )
+ cpu_relax();
+ }
}
panic("FATAL TRAP: vector = %d (%s)\n"
@@ -1711,7 +1739,7 @@ void do_page_fault(struct cpu_user_regs
{
console_start_sync();
printk("Xen SM%cP violation\n", (pf_type == smep_fault) ? 'E' : 'A');
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
if ( pf_type != real_fault )
@@ -1782,7 +1810,7 @@ void __init do_early_page_fault(struct c
console_start_sync();
printk("Early fatal page fault at %04x:%p (cr2=%p, ec=%04x)\n",
regs->cs, _p(regs->eip), _p(cr2), regs->error_code);
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
}
@@ -3598,7 +3626,7 @@ static void pci_serr_error(const struct
default: /* 'fatal' */
console_force_unlock();
printk("\n\nNMI - PCI system error (SERR)\n");
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
}
@@ -3613,7 +3641,7 @@ static void io_check_error(const struct
default: /* 'fatal' */
console_force_unlock();
printk("\n\nNMI - I/O ERROR\n");
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
outb((inb(0x61) & 0x0f) | 0x08, 0x61); /* clear-and-disable IOCK */
@@ -3633,7 +3661,7 @@ static void unknown_nmi_error(const stru
console_force_unlock();
printk("Uhhuh. NMI received for unknown reason %02x.\n", reason);
printk("Do you have a strange power saving mode enabled?\n");
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
}
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -558,6 +558,7 @@ exception_with_ints_disabled:
/* No special register assumptions. */
FATAL_exception_with_ints_disabled:
+ xorl %esi,%esi
movq %rsp,%rdi
call fatal_trap
BUG /* fatal_trap() shouldn't return. */
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -541,7 +541,7 @@ void show_registers(const struct cpu_use
void show_execution_state(const struct cpu_user_regs *regs);
#define dump_execution_state() run_in_exception_handler(show_execution_state)
void show_page_walk(unsigned long addr);
-void noreturn fatal_trap(const struct cpu_user_regs *regs);
+void noreturn fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote);
void compat_show_guest_stack(struct vcpu *v,
const struct cpu_user_regs *regs, int lines);
[-- Attachment #2: x86-NMI-show-global-state.patch --]
[-- Type: text/plain, Size: 6090 bytes --]
x86: show remote CPU state upon fatal NMI or unknown MCE
Quite frequently the watchdog would hit an innocent CPU, e.g. one
trying to acquire a spin lock a remote CPU holds for extended periods
of time, or a random CPU in TSC calbration rendezvous. In such cases
the register and stack dump for that CPU doesn't really help in the
analysis of the problem.
To keep things reasonable on large systems, only log CS:RIP by default.
This can be overridden via a new command line option such that full
register/stack state would get logged.
Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v2: Pass flag to fatal_trap().
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -198,6 +198,14 @@ Permit Xen to use Address Space Identifi
tags the TLB entries with an ID per vcpu. This allows for guest TLB flushes
to be performed without the overhead of a complete TLB flush.
+### async-show-all
+> `= <boolean>`
+
+> Default: `false`
+
+Forces all CPUs' full state to be logged upon certain fatal asynchronous
+exceptions (watchdog NMIs and unexpected MCEs).
+
### ats
> `= <boolean>`
--- a/xen/arch/x86/cpu/mcheck/mce.c
+++ b/xen/arch/x86/cpu/mcheck/mce.c
@@ -77,7 +77,7 @@ static void unexpected_machine_check(con
{
console_force_unlock();
printk("Unexpected Machine Check Exception\n");
- fatal_trap(regs);
+ fatal_trap(regs, 1);
}
--- a/xen/arch/x86/nmi.c
+++ b/xen/arch/x86/nmi.c
@@ -488,7 +488,7 @@ bool_t nmi_watchdog_tick(const struct cp
console_force_unlock();
printk("Watchdog timer detects that CPU%d is stuck!\n",
smp_processor_id());
- fatal_trap(regs);
+ fatal_trap(regs, 1);
}
}
else
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -525,6 +525,25 @@ void vcpu_show_execution_state(struct vc
vcpu_unpause(v);
}
+static cpumask_t show_state_mask;
+static bool_t opt_show_all;
+boolean_param("async-show-all", opt_show_all);
+
+static int nmi_show_execution_state(const struct cpu_user_regs *regs, int cpu)
+{
+ if ( !cpumask_test_cpu(cpu, &show_state_mask) )
+ return 0;
+
+ if ( opt_show_all )
+ show_execution_state(regs);
+ else
+ printk(XENLOG_ERR "CPU%d @ %04x:%08lx (%pS)\n", cpu, regs->cs, regs->rip,
+ guest_mode(regs) ? _p(regs->rip) : NULL);
+ cpumask_clear_cpu(cpu, &show_state_mask);
+
+ return 1;
+}
+
static const char *trapstr(unsigned int trapnr)
{
static const char * const strings[] = {
@@ -544,7 +563,7 @@ static const char *trapstr(unsigned int
* are disabled). In such situations we can't do much that is safe. We try to
* print out some tracing and then we just spin.
*/
-void fatal_trap(const struct cpu_user_regs *regs)
+void fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote)
{
static DEFINE_PER_CPU(char, depth);
unsigned int trapnr = regs->entry_vector;
@@ -570,6 +589,15 @@ void fatal_trap(const struct cpu_user_re
printk("Faulting linear address: %p\n", _p(cr2));
show_page_walk(cr2);
}
+ if ( show_remote )
+ {
+ cpumask_andnot(&show_state_mask, &cpu_online_map,
+ cpumask_of(smp_processor_id()));
+ set_nmi_callback(nmi_show_execution_state);
+ smp_send_nmi_allbutself();
+ while ( !cpumask_empty(&show_state_mask) )
+ cpu_relax();
+ }
}
panic("FATAL TRAP: vector = %d (%s)\n"
@@ -1711,7 +1739,7 @@ void do_page_fault(struct cpu_user_regs
{
console_start_sync();
printk("Xen SM%cP violation\n", (pf_type == smep_fault) ? 'E' : 'A');
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
if ( pf_type != real_fault )
@@ -1782,7 +1810,7 @@ void __init do_early_page_fault(struct c
console_start_sync();
printk("Early fatal page fault at %04x:%p (cr2=%p, ec=%04x)\n",
regs->cs, _p(regs->eip), _p(cr2), regs->error_code);
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
}
@@ -3598,7 +3626,7 @@ static void pci_serr_error(const struct
default: /* 'fatal' */
console_force_unlock();
printk("\n\nNMI - PCI system error (SERR)\n");
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
}
@@ -3613,7 +3641,7 @@ static void io_check_error(const struct
default: /* 'fatal' */
console_force_unlock();
printk("\n\nNMI - I/O ERROR\n");
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
outb((inb(0x61) & 0x0f) | 0x08, 0x61); /* clear-and-disable IOCK */
@@ -3633,7 +3661,7 @@ static void unknown_nmi_error(const stru
console_force_unlock();
printk("Uhhuh. NMI received for unknown reason %02x.\n", reason);
printk("Do you have a strange power saving mode enabled?\n");
- fatal_trap(regs);
+ fatal_trap(regs, 0);
}
}
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -558,6 +558,7 @@ exception_with_ints_disabled:
/* No special register assumptions. */
FATAL_exception_with_ints_disabled:
+ xorl %esi,%esi
movq %rsp,%rdi
call fatal_trap
BUG /* fatal_trap() shouldn't return. */
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -541,7 +541,7 @@ void show_registers(const struct cpu_use
void show_execution_state(const struct cpu_user_regs *regs);
#define dump_execution_state() run_in_exception_handler(show_execution_state)
void show_page_walk(unsigned long addr);
-void noreturn fatal_trap(const struct cpu_user_regs *regs);
+void noreturn fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote);
void compat_show_guest_stack(struct vcpu *v,
const struct cpu_user_regs *regs, int lines);
[-- Attachment #3: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: [PATCH v2] x86: show remote CPU state upon fatal NMI or unknown MCE
2016-06-16 11:26 [PATCH v2] x86: show remote CPU state upon fatal NMI or unknown MCE Jan Beulich
@ 2016-06-16 11:32 ` Andrew Cooper
0 siblings, 0 replies; 2+ messages in thread
From: Andrew Cooper @ 2016-06-16 11:32 UTC (permalink / raw)
To: Jan Beulich, xen-devel
On 16/06/16 12:26, Jan Beulich wrote:
> @@ -570,6 +589,15 @@ void fatal_trap(const struct cpu_user_re
> printk("Faulting linear address: %p\n", _p(cr2));
> show_page_walk(cr2);
> }
> + if ( show_remote )
> + {
> + cpumask_andnot(&show_state_mask, &cpu_online_map,
> + cpumask_of(smp_processor_id()));
> + set_nmi_callback(nmi_show_execution_state);
> + smp_send_nmi_allbutself();
> + while ( !cpumask_empty(&show_state_mask) )
> + cpu_relax();
Sorry about not noticing this before. We need a maximum timeout here,
just like the nmi shootdown, in case one of the other cpus is already
stuck in an NMI handler, so we block the reboot/kexec.
With that fixed, Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2016-06-16 11:32 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-06-16 11:26 [PATCH v2] x86: show remote CPU state upon fatal NMI or unknown MCE Jan Beulich
2016-06-16 11:32 ` Andrew Cooper
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).