[PATCH V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[PATCH V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Anshuman Khandual]

This patch enables facility unavailable exceptions for generic facility,
FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
newly added IRQ statistical counters as and when these exceptions happen.
This also adds couple of helper functions which will be called from within
the interrupt handler context to update their statistics. Similarly this
patch also enables alignment and program check exceptions as well.

	With this patch being applied, /proc/interrupts looks something
like this after running various workloads which create these exceptions.

--------------------------------------------------------------
           CPU0       CPU1
 16:       5734      24129      XICS   2 Level     IPI
 17:          0          0      XICS 4101 Level     virtio0
 18:          0          0      XICS 4100 Level     ohci_hcd:usb1
 19:      13920          0      XICS 4099 Level     virtio1
 20:          0          0      XICS 4096 Level     RAS_EPOW
 21:       6160       3241      XICS 4102 Level     ibmvscsi
 22:          1          0      XICS 4103 Level     hvc_console
LOC:       6825       3556   Local timer interrupts for timer event device
LOC:         22         41   Local timer interrupts for others
SPU:          1          0   Spurious interrupts
PMI:          0          0   Performance monitoring interrupts
MCE:          0          0   Machine check exceptions
DBL:          0          0   Doorbell interrupts
ALN:          0          0   Alignment exceptions
PRG:          0          0   Program check exceptions
FAC:          7         14   Facility unavailable exceptions
FPU:       2928       3162   FPU unavailable exceptions
ALT:      12950      15536   AltiVec unavailable exceptions
VSX:      12930     220183   VSX unavailable exceptions
--------------------------------------------------------------

Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
---
Changes in V3:
- Changed the display string from "ALTIVEC" to "AltiVec"
- Now captured "Facility unavailable exceptions" in the example

Changes in V2:
- Fixed some typos in the final /proc/interrupts output
- Added support for alignment and program check exceptions

 arch/powerpc/include/asm/hardirq.h   |  6 ++++++
 arch/powerpc/kernel/exceptions-64s.S |  2 ++
 arch/powerpc/kernel/irq.c            | 35 +++++++++++++++++++++++++++++++++++
 arch/powerpc/kernel/traps.c          | 28 ++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+)

diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h
index 8add8b8..ba51d3e 100644
--- a/arch/powerpc/include/asm/hardirq.h
+++ b/arch/powerpc/include/asm/hardirq.h
@@ -15,6 +15,12 @@ typedef struct {
 #ifdef CONFIG_PPC_DOORBELL
 	unsigned int doorbell_irqs;
 #endif
+	unsigned int alignment_exceptions;
+	unsigned int program_exceptions;
+	unsigned int fac_unav_exceptions;
+	unsigned int fpu_unav_exceptions;
+	unsigned int altivec_unav_exceptions;
+	unsigned int vsx_unav_exceptions;
 } ____cacheline_aligned irq_cpustat_t;
 
 DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 0a0399c2..a86180c 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
 END_FTR_SECTION_IFSET(CPU_FTR_TM)
 #endif
 	bl	load_up_fpu
+	bl	fpu_unav_exceptions_count
 	b	fast_exception_return
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 2:	/* User process was in a transaction */
@@ -1184,6 +1185,7 @@ BEGIN_FTR_SECTION
   END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
 #endif
 	bl	load_up_altivec
+	bl	altivec_unav_exceptions_count
 	b	fast_exception_return
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 2:	/* User process was in a transaction */
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 4509603..60773b3 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -397,6 +397,35 @@ int arch_show_interrupts(struct seq_file *p, int prec)
 		seq_printf(p, "  Doorbell interrupts\n");
 	}
 #endif
+	seq_printf(p, "%*s: ", prec, "ALN");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", per_cpu(irq_stat, j).alignment_exceptions);
+	seq_printf(p, "  Alignment exceptions\n");
+
+	seq_printf(p, "%*s: ", prec, "PRG");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", per_cpu(irq_stat, j).program_exceptions);
+	seq_printf(p, "  Program check exceptions\n");
+
+	seq_printf(p, "%*s: ", prec, "FAC");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", per_cpu(irq_stat, j).fac_unav_exceptions);
+	seq_printf(p, "  Facility unavailable exceptions\n");
+
+	seq_printf(p, "%*s: ", prec, "FPU");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", per_cpu(irq_stat, j).fpu_unav_exceptions);
+	seq_printf(p, "  FPU unavailable exceptions\n");
+
+	seq_printf(p, "%*s: ", prec, "ALT");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", per_cpu(irq_stat, j).altivec_unav_exceptions);
+	seq_printf(p, "  AltiVec unavailable exceptions\n");
+
+	seq_printf(p, "%*s: ", prec, "VSX");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", per_cpu(irq_stat, j).vsx_unav_exceptions);
+	seq_printf(p, "  VSX unavailable exceptions\n");
 
 	return 0;
 }
@@ -416,6 +445,12 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
 #ifdef CONFIG_PPC_DOORBELL
 	sum += per_cpu(irq_stat, cpu).doorbell_irqs;
 #endif
+	sum += per_cpu(irq_stat, cpu).alignment_exceptions;
+	sum += per_cpu(irq_stat, cpu).program_exceptions;
+	sum += per_cpu(irq_stat, cpu).fac_unav_exceptions;
+	sum += per_cpu(irq_stat, cpu).fpu_unav_exceptions;
+	sum += per_cpu(irq_stat, cpu).altivec_unav_exceptions;
+	sum += per_cpu(irq_stat, cpu).vsx_unav_exceptions;
 
 	return sum;
 }
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 37de90f..d6248c4 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1139,6 +1139,8 @@ void __kprobes program_check_exception(struct pt_regs *regs)
 	enum ctx_state prev_state = exception_enter();
 	unsigned int reason = get_reason(regs);
 
+	__this_cpu_inc(irq_stat.program_exceptions);
+
 	/* We can now get here via a FP Unavailable exception if the core
 	 * has no FPU, in that case the reason flags will be 0 */
 
@@ -1262,6 +1264,8 @@ void alignment_exception(struct pt_regs *regs)
 	enum ctx_state prev_state = exception_enter();
 	int sig, code, fixed = 0;
 
+	__this_cpu_inc(irq_stat.alignment_exceptions);
+
 	/* We restore the interrupt state now */
 	if (!arch_irq_disabled_regs(regs))
 		local_irq_enable();
@@ -1324,6 +1328,8 @@ void kernel_fp_unavailable_exception(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
 
+	__this_cpu_inc(irq_stat.fpu_unav_exceptions);
+
 	printk(KERN_EMERG "Unrecoverable FP Unavailable Exception "
 			  "%lx at %lx\n", regs->trap, regs->nip);
 	die("Unrecoverable FP Unavailable Exception", regs, SIGABRT);
@@ -1335,6 +1341,8 @@ void altivec_unavailable_exception(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
 
+	__this_cpu_inc(irq_stat.altivec_unav_exceptions);
+
 	if (user_mode(regs)) {
 		/* A user program has executed an altivec instruction,
 		   but this kernel doesn't support altivec. */
@@ -1352,6 +1360,8 @@ bail:
 
 void vsx_unavailable_exception(struct pt_regs *regs)
 {
+	__this_cpu_inc(irq_stat.vsx_unav_exceptions);
+
 	if (user_mode(regs)) {
 		/* A user program has executed an vsx instruction,
 		   but this kernel doesn't support vsx. */
@@ -1383,6 +1393,8 @@ void facility_unavailable_exception(struct pt_regs *regs)
 	u8 status;
 	bool hv;
 
+	__this_cpu_inc(irq_stat.fac_unav_exceptions);
+
 	hv = (regs->trap == 0xf80);
 	if (hv)
 		value = mfspr(SPRN_HFSCR);
@@ -1455,10 +1467,22 @@ void facility_unavailable_exception(struct pt_regs *regs)
 }
 #endif
 
+void fpu_unav_exceptions_count(void)
+{
+	__this_cpu_inc(irq_stat.fpu_unav_exceptions);
+}
+
+void altivec_unav_exceptions_count(void)
+{
+	__this_cpu_inc(irq_stat.altivec_unav_exceptions);
+}
+
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 
 void fp_unavailable_tm(struct pt_regs *regs)
 {
+	__this_cpu_inc(irq_stat.fpu_unav_exceptions);
+
 	/* Note:  This does not handle any kind of FP laziness. */
 
 	TM_DEBUG("FP Unavailable trap whilst transactional at 0x%lx, MSR=%lx\n",
@@ -1495,6 +1519,8 @@ void fp_unavailable_tm(struct pt_regs *regs)
 
 void altivec_unavailable_tm(struct pt_regs *regs)
 {
+	__this_cpu_inc(irq_stat.altivec_unav_exceptions);
+
 	/* See the comments in fp_unavailable_tm().  This function operates
 	 * the same way.
 	 */
@@ -1517,6 +1543,8 @@ void vsx_unavailable_tm(struct pt_regs *regs)
 {
 	unsigned long orig_msr = regs->msr;
 
+	__this_cpu_inc(irq_stat.vsx_unav_exceptions);
+
 	/* See the comments in fp_unavailable_tm().  This works similarly,
 	 * though we're loading both FP and VEC registers in here.
 	 *
-- 
2.1.0

Re: [PATCH V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Anshuman Khandual]

On 07/13/2015 01:46 PM, Anshuman Khandual wrote:
> This patch enables facility unavailable exceptions for generic facility,
> FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
> newly added IRQ statistical counters as and when these exceptions happen.
> This also adds couple of helper functions which will be called from within
> the interrupt handler context to update their statistics. Similarly this
> patch also enables alignment and program check exceptions as well.
> 
> 	With this patch being applied, /proc/interrupts looks something
> like this after running various workloads which create these exceptions.
> 
> --------------------------------------------------------------
>            CPU0       CPU1
>  16:       5734      24129      XICS   2 Level     IPI
>  17:          0          0      XICS 4101 Level     virtio0
>  18:          0          0      XICS 4100 Level     ohci_hcd:usb1
>  19:      13920          0      XICS 4099 Level     virtio1
>  20:          0          0      XICS 4096 Level     RAS_EPOW
>  21:       6160       3241      XICS 4102 Level     ibmvscsi
>  22:          1          0      XICS 4103 Level     hvc_console
> LOC:       6825       3556   Local timer interrupts for timer event device
> LOC:         22         41   Local timer interrupts for others
> SPU:          1          0   Spurious interrupts
> PMI:          0          0   Performance monitoring interrupts
> MCE:          0          0   Machine check exceptions
> DBL:          0          0   Doorbell interrupts
> ALN:          0          0   Alignment exceptions
> PRG:          0          0   Program check exceptions
> FAC:          7         14   Facility unavailable exceptions
> FPU:       2928       3162   FPU unavailable exceptions
> ALT:      12950      15536   AltiVec unavailable exceptions
> VSX:      12930     220183   VSX unavailable exceptions
> --------------------------------------------------------------
> 
> Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
> ---
> Changes in V3:
> - Changed the display string from "ALTIVEC" to "AltiVec"
> - Now captured "Facility unavailable exceptions" in the example

Michael/Mikey,

I had already posted the performance comparison details with the
patch in the previous version mail thread. Does this new version
look good enough ?

Re: [V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Michael Ellerman]

On Mon, 2015-13-07 at 08:16:06 UTC, Anshuman Khandual wrote:
> This patch enables facility unavailable exceptions for generic facility,
> FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
> newly added IRQ statistical counters as and when these exceptions happen.
> This also adds couple of helper functions which will be called from within
> the interrupt handler context to update their statistics. Similarly this
> patch also enables alignment and program check exceptions as well.

...

> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> index 0a0399c2..a86180c 100644
> --- a/arch/powerpc/kernel/exceptions-64s.S
> +++ b/arch/powerpc/kernel/exceptions-64s.S
> @@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
>  END_FTR_SECTION_IFSET(CPU_FTR_TM)
>  #endif
>  	bl	load_up_fpu
> +	bl	fpu_unav_exceptions_count

Is it safe to call C code here?

>  	b	fast_exception_return
>  #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
>  2:	/* User process was in a transaction */


cheers

Re: [V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Anshuman Khandual]

On 08/04/2015 03:27 PM, Michael Ellerman wrote:
> On Mon, 2015-13-07 at 08:16:06 UTC, Anshuman Khandual wrote:
>> This patch enables facility unavailable exceptions for generic facility,
>> FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
>> newly added IRQ statistical counters as and when these exceptions happen.
>> This also adds couple of helper functions which will be called from within
>> the interrupt handler context to update their statistics. Similarly this
>> patch also enables alignment and program check exceptions as well.
> 
> ...
> 
>> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
>> index 0a0399c2..a86180c 100644
>> --- a/arch/powerpc/kernel/exceptions-64s.S
>> +++ b/arch/powerpc/kernel/exceptions-64s.S
>> @@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
>>  END_FTR_SECTION_IFSET(CPU_FTR_TM)
>>  #endif
>>  	bl	load_up_fpu
>> +	bl	fpu_unav_exceptions_count
> 
> Is it safe to call C code here?

Hmm, is it not ? I had that question but was not really sure. Dont
understand the difference between 'fast_exception_return' and
'ret_from_except' completely. Will converting the following sequence
of code

 	bl	load_up_fpu
+	bl	fpu_unav_exceptions_count
 	b	fast_exception_return

into

 	bl	load_up_fpu
        RECONCILE_IRQ_STATE(r10, r11)
        addi    r3,r1,STACK_FRAME_OVERHEAD
+	bl	fpu_unav_exceptions_count
 	b	ret_from_except

help solve the problem ?

Re: [V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Benjamin Herrenschmidt]

On Tue, 2015-08-04 at 19:57 +1000, Michael Ellerman wrote:
> On Mon, 2015-13-07 at 08:16:06 UTC, Anshuman Khandual wrote:
> > This patch enables facility unavailable exceptions for generic facility,
> > FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
> > newly added IRQ statistical counters as and when these exceptions happen.
> > This also adds couple of helper functions which will be called from within
> > the interrupt handler context to update their statistics. Similarly this
> > patch also enables alignment and program check exceptions as well.
> 
> ...
> 
> > diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> > index 0a0399c2..a86180c 100644
> > --- a/arch/powerpc/kernel/exceptions-64s.S
> > +++ b/arch/powerpc/kernel/exceptions-64s.S
> > @@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
> >  END_FTR_SECTION_IFSET(CPU_FTR_TM)
> >  #endif
> >  	bl	load_up_fpu
> > +	bl	fpu_unav_exceptions_count
> 
> Is it safe to call C code here?

Even if it was (at some stage it wasn't, I'd have to look very closely
to see what's the situation now), we certainly don't want to add
overhead to load_up_fpu.

> >  	b	fast_exception_return
> >  #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
> >  2:	/* User process was in a transaction */
> 
> 
> cheers
> _______________________________________________
> Linuxppc-dev mailing list
> Linuxppc-dev@lists.ozlabs.org
> https://lists.ozlabs.org/listinfo/linuxppc-dev

Re: [V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Anshuman Khandual]

On 08/09/2015 07:57 AM, Benjamin Herrenschmidt wrote:
> On Tue, 2015-08-04 at 19:57 +1000, Michael Ellerman wrote:
>> > On Mon, 2015-13-07 at 08:16:06 UTC, Anshuman Khandual wrote:
>>> > > This patch enables facility unavailable exceptions for generic facility,
>>> > > FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
>>> > > newly added IRQ statistical counters as and when these exceptions happen.
>>> > > This also adds couple of helper functions which will be called from within
>>> > > the interrupt handler context to update their statistics. Similarly this
>>> > > patch also enables alignment and program check exceptions as well.
>> > 
>> > ...
>> > 
>>> > > diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
>>> > > index 0a0399c2..a86180c 100644
>>> > > --- a/arch/powerpc/kernel/exceptions-64s.S
>>> > > +++ b/arch/powerpc/kernel/exceptions-64s.S
>>> > > @@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
>>> > >  END_FTR_SECTION_IFSET(CPU_FTR_TM)
>>> > >  #endif
>>> > >  	bl	load_up_fpu
>>> > > +	bl	fpu_unav_exceptions_count
>> > 
>> > Is it safe to call C code here?
> Even if it was (at some stage it wasn't, I'd have to look very closely
> to see what's the situation now), we certainly don't want to add
> overhead to load_up_fpu.

As I had already mentioned in the V2 thread of this patch, the
FPU performance with this patch being applied is still very much
comparable to the kernel without this patch. Though I have not
verified whether this still holds true with the new changes being
proposed in exceptions-64s.S (earlier reply in this thread) to
make the C function call safer.

Average of 1000 iterations (context_switch2 --fp 0 0)

With    the patch : 322599.57  (Average of 1000 results)
Without the patch : 320464.924 (Average of 1000 results)

With standard deviation of the results.

6029.1407073288 (with patch ) 5941.7684079774 (without patch)

Wondering if the result above still does not convince us
that FPU performance might not be getting hit because of
this patch, let me know if we need to do more experiments.

Re: [V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Michael Ellerman]

On Thu, 2015-08-06 at 18:54 +0530, Anshuman Khandual wrote:
> On 08/04/2015 03:27 PM, Michael Ellerman wrote:
> > On Mon, 2015-13-07 at 08:16:06 UTC, Anshuman Khandual wrote:
> >> This patch enables facility unavailable exceptions for generic facility,
> >> FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
> >> newly added IRQ statistical counters as and when these exceptions happen.
> >> This also adds couple of helper functions which will be called from within
> >> the interrupt handler context to update their statistics. Similarly this
> >> patch also enables alignment and program check exceptions as well.
> > 
> > ...
> > 
> >> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> >> index 0a0399c2..a86180c 100644
> >> --- a/arch/powerpc/kernel/exceptions-64s.S
> >> +++ b/arch/powerpc/kernel/exceptions-64s.S
> >> @@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
> >>  END_FTR_SECTION_IFSET(CPU_FTR_TM)
> >>  #endif
> >>  	bl	load_up_fpu
> >> +	bl	fpu_unav_exceptions_count
> > 
> > Is it safe to call C code here?
> 
> Hmm, is it not ? I had that question but was not really sure. Dont
> understand the difference between 'fast_exception_return' and
> 'ret_from_except' completely.

If you're "not really sure" it's correct, please say so in the change log!

I'd rather you didn't send me patches with possibly subtle bugs in core code.

cheers

Re: [V3] powerpc/irq: Enable some more exceptions in /proc/interrupts interface

[Anshuman Khandual]

On 08/14/2015 08:22 AM, Michael Ellerman wrote:
> On Thu, 2015-08-06 at 18:54 +0530, Anshuman Khandual wrote:
>> On 08/04/2015 03:27 PM, Michael Ellerman wrote:
>>> On Mon, 2015-13-07 at 08:16:06 UTC, Anshuman Khandual wrote:
>>>> This patch enables facility unavailable exceptions for generic facility,
>>>> FPU, ALTIVEC and VSX in /proc/interrupts listing by incrementing their
>>>> newly added IRQ statistical counters as and when these exceptions happen.
>>>> This also adds couple of helper functions which will be called from within
>>>> the interrupt handler context to update their statistics. Similarly this
>>>> patch also enables alignment and program check exceptions as well.
>>>
>>> ...
>>>
>>>> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
>>>> index 0a0399c2..a86180c 100644
>>>> --- a/arch/powerpc/kernel/exceptions-64s.S
>>>> +++ b/arch/powerpc/kernel/exceptions-64s.S
>>>> @@ -1158,6 +1158,7 @@ BEGIN_FTR_SECTION
>>>>  END_FTR_SECTION_IFSET(CPU_FTR_TM)
>>>>  #endif
>>>>  	bl	load_up_fpu
>>>> +	bl	fpu_unav_exceptions_count
>>>
>>> Is it safe to call C code here?
>>
>> Hmm, is it not ? I had that question but was not really sure. Dont
>> understand the difference between 'fast_exception_return' and
>> 'ret_from_except' completely.
> 
> If you're "not really sure" it's correct, please say so in the change log!

Yeah I should have written that up some where after the commit
message (after "---"). Its my bad, will take care of this next
time around.

> 
> I'd rather you didn't send me patches with possibly subtle bugs in core code.

Michael, I understand your concern. I was just trying to add
new entries in there which would help us. Wondering whats our
plan for this patch, if we change it as I had proposed earlier,
will it be good enough.