[PATCH 0/2] perf: Use LBR for machine/oops debugging

[PATCH 0/2] perf: Use LBR for machine/oops debugging

[Peter Zijlstra]

The LBRs are relatively cheap to keep enabled and provide some history
to OOPSen, also some CPUs are reported to keep them over soft-reset,
which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable
the LBRs and will print the LBRs on CPU init and die().

[PATCH 1/2] perf, x86: fix __initconst vs const

[Peter Zijlstra]

[-- Attachment #1: perf-initconst.patch --]
[-- Type: text/plain, Size: 4642 bytes --]

All variables that have __initconst should also be const.

Suggested-by: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 arch/x86/kernel/cpu/perf_event_amd.c   |    4 ++--
 arch/x86/kernel/cpu/perf_event_intel.c |   12 ++++++------
 arch/x86/kernel/cpu/perf_event_p4.c    |    4 ++--
 arch/x86/kernel/cpu/perf_event_p6.c    |    2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

Index: linux-2.6/arch/x86/kernel/cpu/perf_event_amd.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_amd.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_amd.c
@@ -2,7 +2,7 @@
 
 static DEFINE_RAW_SPINLOCK(amd_nb_lock);
 
-static __initconst u64 amd_hw_cache_event_ids
+static __initconst const u64 amd_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -371,7 +371,7 @@ static void amd_pmu_cpu_dead(int cpu)
 	raw_spin_unlock(&amd_nb_lock);
 }
 
-static __initconst struct x86_pmu amd_pmu = {
+static __initconst const struct x86_pmu amd_pmu = {
 	.name			= "AMD",
 	.handle_irq		= x86_pmu_handle_irq,
 	.disable_all		= x86_pmu_disable_all,
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
@@ -88,7 +88,7 @@ static u64 intel_pmu_event_map(int hw_ev
 	return intel_perfmon_event_map[hw_event];
 }
 
-static __initconst u64 westmere_hw_cache_event_ids
+static __initconst const u64 westmere_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -179,7 +179,7 @@ static __initconst u64 westmere_hw_cache
  },
 };
 
-static __initconst u64 nehalem_hw_cache_event_ids
+static __initconst const u64 nehalem_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -270,7 +270,7 @@ static __initconst u64 nehalem_hw_cache_
  },
 };
 
-static __initconst u64 core2_hw_cache_event_ids
+static __initconst const u64 core2_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -361,7 +361,7 @@ static __initconst u64 core2_hw_cache_ev
  },
 };
 
-static __initconst u64 atom_hw_cache_event_ids
+static __initconst const u64 atom_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -776,7 +776,7 @@ intel_get_event_constraints(struct cpu_h
 	return x86_get_event_constraints(cpuc, event);
 }
 
-static __initconst struct x86_pmu core_pmu = {
+static __initconst const struct x86_pmu core_pmu = {
 	.name			= "core",
 	.handle_irq		= x86_pmu_handle_irq,
 	.disable_all		= x86_pmu_disable_all,
@@ -815,7 +815,7 @@ static void intel_pmu_cpu_dying(int cpu)
 	fini_debug_store_on_cpu(cpu);
 }
 
-static __initconst struct x86_pmu intel_pmu = {
+static __initconst const struct x86_pmu intel_pmu = {
 	.name			= "Intel",
 	.handle_irq		= intel_pmu_handle_irq,
 	.disable_all		= intel_pmu_disable_all,
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_p4.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_p4.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_p4.c
@@ -287,7 +287,7 @@ static struct p4_event_bind p4_event_bin
 	p4_config_pack_cccr(cache_event					| \
 			    P4_CCCR_ESEL(P4_OPCODE_ESEL(P4_OPCODE(event))))
 
-static __initconst u64 p4_hw_cache_event_ids
+static __initconst const u64 p4_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -775,7 +775,7 @@ done:
 	return num ? -ENOSPC : 0;
 }
 
-static __initconst struct x86_pmu p4_pmu = {
+static __initconst const struct x86_pmu p4_pmu = {
 	.name			= "Netburst P4/Xeon",
 	.handle_irq		= p4_pmu_handle_irq,
 	.disable_all		= p4_pmu_disable_all,
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_p6.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_p6.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_p6.c
@@ -102,7 +102,7 @@ static void p6_pmu_enable_event(struct p
 	(void)checking_wrmsrl(hwc->config_base + hwc->idx, val);
 }
 
-static __initconst struct x86_pmu p6_pmu = {
+static __initconst const struct x86_pmu p6_pmu = {
 	.name			= "p6",
 	.handle_irq		= x86_pmu_handle_irq,
 	.disable_all		= p6_pmu_disable_all,

[RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Peter Zijlstra]

[-- Attachment #1: perf-lbr-debug.patch --]
[-- Type: text/plain, Size: 5414 bytes --]

The LBRs are relatively cheap to keep enabled and provide some history
to OOPSen, also some CPUs are reported to keep them over soft-reset,
which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable
the LBRs and will print the LBRs on CPU init and die().

Requested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 arch/x86/include/asm/perf_event.h          |    7 ++
 arch/x86/kernel/cpu/perf_event_intel.c     |    5 -
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   86 +++++++++++++++++++++++++++--
 arch/x86/kernel/dumpstack.c                |    5 +
 4 files changed, 95 insertions(+), 8 deletions(-)

Index: linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/perf_event.h
+++ linux-2.6/arch/x86/include/asm/perf_event.h
@@ -155,9 +155,14 @@ extern void perf_events_lapic_init(void)
 
 #define perf_instruction_pointer(regs)	((regs)->ip)
 
+void dump_lbr_state(void);
+void lbr_off(void);
+
 #else
 static inline void init_hw_perf_events(void)		{ }
-static inline void perf_events_lapic_init(void)	{ }
+static inline void perf_events_lapic_init(void)		{ }
+static inline void dump_lbr_state(void)			{ }
+static inline void lbr_off(void)			{ }
 #endif
 
 #endif /* _ASM_X86_PERF_EVENT_H */
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
@@ -804,10 +804,7 @@ static __initconst const struct x86_pmu 
 static void intel_pmu_cpu_starting(int cpu)
 {
 	init_debug_store_on_cpu(cpu);
-	/*
-	 * Deal with CPUs that don't clear their LBRs on power-up.
-	 */
-	intel_pmu_lbr_reset();
+	intel_pmu_lbr_starting();
 }
 
 static void intel_pmu_cpu_dying(int cpu)
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel_lbr.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -1,12 +1,32 @@
 #ifdef CONFIG_CPU_SUP_INTEL
 
 enum {
+	LBR_DEBUG_OFF		= 0,
+	LBR_DEBUG_ON		= 1,
+};
+
+static int lbr_debug_state __read_mostly;
+
+static int __init setup_lbr_debug(char *str)
+{
+	if (!strcmp(str, "on"))
+		lbr_debug_state = LBR_DEBUG_ON;
+	else
+		return 0;
+	return 1;
+}
+
+__setup("lbr_debug=", setup_lbr_debug);
+
+enum {
 	LBR_FORMAT_32		= 0x00,
 	LBR_FORMAT_LIP		= 0x01,
 	LBR_FORMAT_EIP		= 0x02,
 	LBR_FORMAT_EIP_FLAGS	= 0x03,
 };
 
+static DEFINE_PER_CPU(int, lbr_print_done);
+
 /*
  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  * otherwise it becomes near impossible to get a reliable stack.
@@ -50,9 +70,6 @@ static void intel_pmu_lbr_reset_64(void)
 
 static void intel_pmu_lbr_reset(void)
 {
-	if (!x86_pmu.lbr_nr)
-		return;
-
 	if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
 		intel_pmu_lbr_reset_32();
 	else
@@ -182,6 +199,8 @@ static void intel_pmu_lbr_read(void)
 {
 	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
+	cpuc->lbr_stack.nr = 0;
+
 	if (!cpuc->lbr_users)
 		return;
 
@@ -215,4 +234,65 @@ static void intel_pmu_lbr_init_atom(void
 	x86_pmu.lbr_to     = 0x60;
 }
 
+static void __dump_lbr_state(void)
+{
+	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+	int i;
+
+	intel_pmu_lbr_read();
+	for (i = 0; i < cpuc->lbr_stack.nr; i++) {
+		printk(KERN_DEBUG "CPU%d LBR%d: %pF -> %pF\n",
+				smp_processor_id(), i,
+				(void *)cpuc->lbr_entries[i].from,
+				(void *)cpuc->lbr_entries[i].to);
+	}
+}
+
+static void intel_pmu_lbr_starting(void)
+{
+	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+	if (!x86_pmu.lbr_nr)
+		return;
+
+	cpuc->lbr_users = 1;
+	if (lbr_debug_state && !__get_cpu_var(lbr_print_done)) {
+		__get_cpu_var(lbr_print_done) = 1;
+		__dump_lbr_state();
+	}
+
+	intel_pmu_lbr_reset();
+
+	if (lbr_debug_state)
+		__intel_pmu_lbr_enable();
+	else
+		cpuc->lbr_users = 0;
+}
+
+void dump_lbr_state(void)
+{
+	if (!lbr_debug_state)
+		return;
+
+	__dump_lbr_state();
+}
+
+void lbr_off(void)
+{
+	if (!lbr_debug_state)
+		return;
+
+	__intel_pmu_lbr_disable();
+}
+
+#else
+
+void dump_lbr_state(void)
+{
+}
+
+void lbr_off(void)
+{
+}
+
 #endif /* CONFIG_CPU_SUP_INTEL */
Index: linux-2.6/arch/x86/kernel/dumpstack.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/dumpstack.c
+++ linux-2.6/arch/x86/kernel/dumpstack.c
@@ -17,6 +17,7 @@
 #include <linux/sysfs.h>
 
 #include <asm/stacktrace.h>
+#include <asm/perf_event.h>
 
 #include "dumpstack.h"
 
@@ -224,6 +225,8 @@ unsigned __kprobes long oops_begin(void)
 	int cpu;
 	unsigned long flags;
 
+	lbr_off();
+
 	oops_enter();
 
 	/* racy, but better than risking deadlock. */
@@ -306,6 +309,7 @@ int __kprobes __die(const char *str, str
 	printk_address(regs->ip, 1);
 	printk(" RSP <%016lx>\n", regs->sp);
 #endif
+	dump_lbr_state();
 	return 0;
 }
 
@@ -343,6 +347,7 @@ die_nmi(char *str, struct pt_regs *regs,
 	printk(" on CPU%d, ip %08lx, registers:\n",
 		smp_processor_id(), regs->ip);
 	show_registers(regs);
+	dump_lbr_state();
 	oops_end(flags, regs, 0);
 	if (do_panic || panic_on_oops)
 		panic("Non maskable interrupt");

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Stephane Eranian]

On Mon, Mar 29, 2010 at 1:20 PM, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> The LBRs are relatively cheap to keep enabled and provide some history
> to OOPSen, also some CPUs are reported to keep them over soft-reset,
> which allows us to use them to debug things like tripple faults.
>
> Therefore introduce a boot option: lbr_debug=on, which always enable
> the LBRs and will print the LBRs on CPU init and die().
>

When this is enabled, it will prevent changing the LBR configuration to
record only selected branches. Unless you are willing to accept filtered
content in the kernel dump.


> Requested-by: Linus Torvalds <torvalds@linux-foundation.org>
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> ---
>  arch/x86/include/asm/perf_event.h          |    7 ++
>  arch/x86/kernel/cpu/perf_event_intel.c     |    5 -
>  arch/x86/kernel/cpu/perf_event_intel_lbr.c |   86 +++++++++++++++++++++++++++--
>  arch/x86/kernel/dumpstack.c                |    5 +
>  4 files changed, 95 insertions(+), 8 deletions(-)
>
> Index: linux-2.6/arch/x86/include/asm/perf_event.h
> ===================================================================
> --- linux-2.6.orig/arch/x86/include/asm/perf_event.h
> +++ linux-2.6/arch/x86/include/asm/perf_event.h
> @@ -155,9 +155,14 @@ extern void perf_events_lapic_init(void)
>
>  #define perf_instruction_pointer(regs) ((regs)->ip)
>
> +void dump_lbr_state(void);
> +void lbr_off(void);
> +
>  #else
>  static inline void init_hw_perf_events(void)           { }
> -static inline void perf_events_lapic_init(void)        { }
> +static inline void perf_events_lapic_init(void)                { }
> +static inline void dump_lbr_state(void)                        { }
> +static inline void lbr_off(void)                       { }
>  #endif
>
>  #endif /* _ASM_X86_PERF_EVENT_H */
> Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
> ===================================================================
> --- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel.c
> +++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
> @@ -804,10 +804,7 @@ static __initconst const struct x86_pmu
>  static void intel_pmu_cpu_starting(int cpu)
>  {
>        init_debug_store_on_cpu(cpu);
> -       /*
> -        * Deal with CPUs that don't clear their LBRs on power-up.
> -        */
> -       intel_pmu_lbr_reset();
> +       intel_pmu_lbr_starting();
>  }
>
>  static void intel_pmu_cpu_dying(int cpu)
> Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel_lbr.c
> ===================================================================
> --- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel_lbr.c
> +++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel_lbr.c
> @@ -1,12 +1,32 @@
>  #ifdef CONFIG_CPU_SUP_INTEL
>
>  enum {
> +       LBR_DEBUG_OFF           = 0,
> +       LBR_DEBUG_ON            = 1,
> +};
> +
> +static int lbr_debug_state __read_mostly;
> +
> +static int __init setup_lbr_debug(char *str)
> +{
> +       if (!strcmp(str, "on"))
> +               lbr_debug_state = LBR_DEBUG_ON;
> +       else
> +               return 0;
> +       return 1;
> +}
> +
> +__setup("lbr_debug=", setup_lbr_debug);
> +
> +enum {
>        LBR_FORMAT_32           = 0x00,
>        LBR_FORMAT_LIP          = 0x01,
>        LBR_FORMAT_EIP          = 0x02,
>        LBR_FORMAT_EIP_FLAGS    = 0x03,
>  };
>
> +static DEFINE_PER_CPU(int, lbr_print_done);
> +
>  /*
>  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
>  * otherwise it becomes near impossible to get a reliable stack.
> @@ -50,9 +70,6 @@ static void intel_pmu_lbr_reset_64(void)
>
>  static void intel_pmu_lbr_reset(void)
>  {
> -       if (!x86_pmu.lbr_nr)
> -               return;
> -
>        if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
>                intel_pmu_lbr_reset_32();
>        else
> @@ -182,6 +199,8 @@ static void intel_pmu_lbr_read(void)
>  {
>        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
>
> +       cpuc->lbr_stack.nr = 0;
> +
>        if (!cpuc->lbr_users)
>                return;
>
> @@ -215,4 +234,65 @@ static void intel_pmu_lbr_init_atom(void
>        x86_pmu.lbr_to     = 0x60;
>  }
>
> +static void __dump_lbr_state(void)
> +{
> +       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
> +       int i;
> +
> +       intel_pmu_lbr_read();
> +       for (i = 0; i < cpuc->lbr_stack.nr; i++) {
> +               printk(KERN_DEBUG "CPU%d LBR%d: %pF -> %pF\n",
> +                               smp_processor_id(), i,
> +                               (void *)cpuc->lbr_entries[i].from,
> +                               (void *)cpuc->lbr_entries[i].to);
> +       }
> +}
> +
> +static void intel_pmu_lbr_starting(void)
> +{
> +       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
> +
> +       if (!x86_pmu.lbr_nr)
> +               return;
> +
> +       cpuc->lbr_users = 1;
> +       if (lbr_debug_state && !__get_cpu_var(lbr_print_done)) {
> +               __get_cpu_var(lbr_print_done) = 1;
> +               __dump_lbr_state();
> +       }
> +
> +       intel_pmu_lbr_reset();
> +
> +       if (lbr_debug_state)
> +               __intel_pmu_lbr_enable();
> +       else
> +               cpuc->lbr_users = 0;
> +}
> +
> +void dump_lbr_state(void)
> +{
> +       if (!lbr_debug_state)
> +               return;
> +
> +       __dump_lbr_state();
> +}
> +
> +void lbr_off(void)
> +{
> +       if (!lbr_debug_state)
> +               return;
> +
> +       __intel_pmu_lbr_disable();
> +}
> +
> +#else
> +
> +void dump_lbr_state(void)
> +{
> +}
> +
> +void lbr_off(void)
> +{
> +}
> +
>  #endif /* CONFIG_CPU_SUP_INTEL */
> Index: linux-2.6/arch/x86/kernel/dumpstack.c
> ===================================================================
> --- linux-2.6.orig/arch/x86/kernel/dumpstack.c
> +++ linux-2.6/arch/x86/kernel/dumpstack.c
> @@ -17,6 +17,7 @@
>  #include <linux/sysfs.h>
>
>  #include <asm/stacktrace.h>
> +#include <asm/perf_event.h>
>
>  #include "dumpstack.h"
>
> @@ -224,6 +225,8 @@ unsigned __kprobes long oops_begin(void)
>        int cpu;
>        unsigned long flags;
>
> +       lbr_off();
> +
>        oops_enter();
>
>        /* racy, but better than risking deadlock. */
> @@ -306,6 +309,7 @@ int __kprobes __die(const char *str, str
>        printk_address(regs->ip, 1);
>        printk(" RSP <%016lx>\n", regs->sp);
>  #endif
> +       dump_lbr_state();
>        return 0;
>  }
>
> @@ -343,6 +347,7 @@ die_nmi(char *str, struct pt_regs *regs,
>        printk(" on CPU%d, ip %08lx, registers:\n",
>                smp_processor_id(), regs->ip);
>        show_registers(regs);
> +       dump_lbr_state();
>        oops_end(flags, regs, 0);
>        if (do_panic || panic_on_oops)
>                panic("Non maskable interrupt");
>
>
>

Re: [PATCH 0/2] perf: Use LBR for machine/oops debugging

[Andi Kleen]

Peter Zijlstra <a.p.zijlstra@chello.nl> writes:

> The LBRs are relatively cheap to keep enabled and provide some history
> to OOPSen, also some CPUs are reported to keep them over soft-reset,
> which allows us to use them to debug things like tripple faults.
>
> Therefore introduce a boot option: lbr_debug=on, which always enable
> the LBRs and will print the LBRs on CPU init and die().


die is too late. they will only contain the oops code then.

-Andi

-- 
ak@linux.intel.com -- Speaking for myself only.

Re: [PATCH 0/2] perf: Use LBR for machine/oops debugging

[Stephane Eranian]

On Mon, Mar 29, 2010 at 2:08 PM, Andi Kleen <andi@firstfloor.org> wrote:
> Peter Zijlstra <a.p.zijlstra@chello.nl> writes:
>
>> The LBRs are relatively cheap to keep enabled and provide some history
>> to OOPSen, also some CPUs are reported to keep them over soft-reset,
>> which allows us to use them to debug things like tripple faults.
>>
>> Therefore introduce a boot option: lbr_debug=on, which always enable
>> the LBRs and will print the LBRs on CPU init and die().
>
>
> die is too late. they will only contain the oops code then.
>
Most likely yes,  especially with small LBR, such as on Intel Core.


> -Andi
>
> --
> ak@linux.intel.com -- Speaking for myself only.
>

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Peter Zijlstra]

On Mon, 2010-03-29 at 13:47 +0100, Stephane Eranian wrote:
> On Mon, Mar 29, 2010 at 1:20 PM, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> > The LBRs are relatively cheap to keep enabled and provide some history
> > to OOPSen, also some CPUs are reported to keep them over soft-reset,
> > which allows us to use them to debug things like tripple faults.
> >
> > Therefore introduce a boot option: lbr_debug=on, which always enable
> > the LBRs and will print the LBRs on CPU init and die().
> >
> 
> When this is enabled, it will prevent changing the LBR configuration to
> record only selected branches. Unless you are willing to accept filtered
> content in the kernel dump. 

Sure, but since we don't support that silly config reg anyway that's
pretty much not an issue ;-)

Re: [PATCH 0/2] perf: Use LBR for machine/oops debugging

[Peter Zijlstra]

On Mon, 2010-03-29 at 15:08 +0200, Andi Kleen wrote:
> Peter Zijlstra <a.p.zijlstra@chello.nl> writes:
> 
> > The LBRs are relatively cheap to keep enabled and provide some history
> > to OOPSen, also some CPUs are reported to keep them over soft-reset,
> > which allows us to use them to debug things like tripple faults.
> >
> > Therefore introduce a boot option: lbr_debug=on, which always enable
> > the LBRs and will print the LBRs on CPU init and die().
> 
> 
> die is too late. they will only contain the oops code then.

We do an lbr_off() in oops_begin(), or is there a better/earlier place
we can do that?

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Stephane Eranian]

On Mon, Mar 29, 2010 at 3:13 PM, Peter Zijlstra <peterz@infradead.org> wrote:
> On Mon, 2010-03-29 at 13:47 +0100, Stephane Eranian wrote:
>> On Mon, Mar 29, 2010 at 1:20 PM, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
>> > The LBRs are relatively cheap to keep enabled and provide some history
>> > to OOPSen, also some CPUs are reported to keep them over soft-reset,
>> > which allows us to use them to debug things like tripple faults.
>> >
>> > Therefore introduce a boot option: lbr_debug=on, which always enable
>> > the LBRs and will print the LBRs on CPU init and die().
>> >
>>
>> When this is enabled, it will prevent changing the LBR configuration to
>> record only selected branches. Unless you are willing to accept filtered
>> content in the kernel dump.
>
> Sure, but since we don't support that silly config reg anyway that's
> pretty much not an issue ;-)
>
I will provide a patch to make it available. This is needed for certain
measurements.

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Ingo Molnar]

* Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:

> The LBRs are relatively cheap to keep enabled and provide some history to 
> OOPSen, also some CPUs are reported to keep them over soft-reset, which 
> allows us to use them to debug things like tripple faults.
> 
> Therefore introduce a boot option: lbr_debug=on, which always enable the 
> LBRs and will print the LBRs on CPU init and die().

Yummie!

Have you got some sample lbr_debug=1 output as well by any chance, with a 
crash provoked somewhere? How good is the output in practice? (i.e. how many 
artificial entries do we have at the end of the buffer, filled with crash 
related addresses?)

Also, i think we should use something more descriptive than lbr_debug=y. 
Perhaps crash_trace=1 or so?

Plus, it would be nice to have a sysctl entry for this as well - so that 
production systems can enable this if they want to enrich the output of some 
difficult-to-analyze kernel crash, without yet another reboot.

	Ingo

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Peter Zijlstra]

On Mon, 2010-03-29 at 19:02 +0200, Ingo Molnar wrote:
> * Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> 
> > The LBRs are relatively cheap to keep enabled and provide some history to 
> > OOPSen, also some CPUs are reported to keep them over soft-reset, which 
> > allows us to use them to debug things like tripple faults.
> > 
> > Therefore introduce a boot option: lbr_debug=on, which always enable the 
> > LBRs and will print the LBRs on CPU init and die().
> 
> Yummie!
> 
> Have you got some sample lbr_debug=1 output as well by any chance, with a 
> crash provoked somewhere? How good is the output in practice? (i.e. how many 
> artificial entries do we have at the end of the buffer, filled with crash 
> related addresses?)

lbr_debug=on actually, =1 doesn't parse. I had some output, but I was
still looking at finding out why my %pF formats for CPU0 didn't have
symbols in, probably a init sequence issue.

> Also, i think we should use something more descriptive than lbr_debug=y. 
> Perhaps crash_trace=1 or so?

Well, I would like to keep LBR in the name, since that is the mechanism
used, we could maybe do lbr_trace or somesuch?

> Plus, it would be nice to have a sysctl entry for this as well - so that 
> production systems can enable this if they want to enrich the output of some 
> difficult-to-analyze kernel crash, without yet another reboot.

Right, could do, but once it crashed it clearly to late to enable
anything ;-)

Re: [PATCH 0/2] perf: Use LBR for machine/oops debugging

[Andi Kleen]

On Mon, Mar 29, 2010 at 04:14:19PM +0200, Peter Zijlstra wrote:
> On Mon, 2010-03-29 at 15:08 +0200, Andi Kleen wrote:
> > Peter Zijlstra <a.p.zijlstra@chello.nl> writes:
> > 
> > > The LBRs are relatively cheap to keep enabled and provide some history
> > > to OOPSen, also some CPUs are reported to keep them over soft-reset,
> > > which allows us to use them to debug things like tripple faults.
> > >
> > > Therefore introduce a boot option: lbr_debug=on, which always enable
> > > the LBRs and will print the LBRs on CPU init and die().
> > 
> > 
> > die is too late. they will only contain the oops code then.
> 
> We do an lbr_off() in oops_begin(), or is there a better/earlier place
> we can do that?

I had an old patch in the P4 era (slightly different but larger LBRs) which
saved them all early in the exception handlers and then dumped
them from the buffer. That's early enough that you only miss one or two.

The problem is that it's somewhat more expensive, the MSR reads
are not cheap and they will slow down all your page faults.

I checked, but I can't find the old patch anymore. Could be probably
redone. 

-Andi
-- 
ak@linux.intel.com -- Speaking for myself only.

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

[Ingo Molnar]

* Peter Zijlstra <peterz@infradead.org> wrote:

> On Mon, 2010-03-29 at 19:02 +0200, Ingo Molnar wrote:
>
> > Plus, it would be nice to have a sysctl entry for this as well - so that 
> > production systems can enable this if they want to enrich the output of 
> > some difficult-to-analyze kernel crash, without yet another reboot.
> 
> Right, could do, but once it crashed it clearly to late to enable anything 
> ;-)

No. What i mean is that with your patch, a debugging session would go like 
this:

  < kernel crashes >                            # reboot #1

  < admin logs in and scratches head >
  < admin consults kernel hackers and enables lbr_debug=1 in /etc/grub.conf >

  < admin reboots >                             # reboot #2

  < kernel crashes again >                      # reboot #3

With the sysctl we'd have one reboot less:

  < kernel crashes >                            # reboot #1

  < admin logs in and scratches head >
  < admin consults kernel hackers and tweaks /proc/sys/kernel/x86/lbr_debug >

  < kernel crashes again >                      # reboot #2

Thanks,

	Ingo