[PATCH 0/9] arm64: add more accurate code for tracing the hardirq flags changes

[PATCH 0/9] arm64: add more accurate code for tracing the hardirq flags changes

[Huang Shijie]

In the current arm64 code, we trace the hardirq flags change occasionally.
Sometimes, the trace code is buggy, such as the el0_irq.

This patch sets 
   0.) in order to avoid the save/restore registers, add patches to adjust
        the code.
   1.) add the trace code at the entry/exit, 
   2.) add the trace code at the irq macros.
   3.) removes the duplicated trace code.

I tested this patch set with CONFIG_PROVE_LOCKING/CONFIG_LOCKDEP/CONFIG_LOCK_STAT
enabled in the Juno-r1 board.

The whole patch set is based on the patch: 
    http://lists.infradead.org/pipermail/linux-arm-kernel/2016-April/420154.html

Thanks to the code review of Steve/James/Mark, I really appriciate it.

Huang Shijie (9):
  arm64: entry: use the callee-saved registers to save some registers
  arm64: entry: add a new macro to restore the registers for syscall
  arm64: entry: record the hardirq changes on the entry/exit code
  arm64: entry: remove the duplicated code for tracing the IRQ flags
  arm64: entry: only restore the syscall registers once
  arm64: entry: adjust el1_sync so that a function can be called
  arm64: entry: save the x0 back into the stack before disabling the
    interrupt
  arm64: add the hardirq flags trace code for irq macros
  arm64: entry: remove the duplicated hardirq flags trace code

 arch/arm64/include/asm/assembler.h |  9 ++++
 arch/arm64/kernel/entry.S          | 88 ++++++++++++++++++++++----------------
 arch/arm64/kernel/signal.c         |  6 ---
 arch/arm64/mm/fault.c              | 10 -----
 4 files changed, 59 insertions(+), 54 deletions(-)

-- 
2.5.5

[PATCH 1/9] arm64: entry: use the callee-saved registers to save some registers

[Huang Shijie]

This patch uses the callee-saved registers x19/x20/scno to save the
x0/x7/x8 which will be used in the syscall path.

This patch makes preparation for the trace code.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index d70a9e4..7dcfd49 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -119,6 +119,16 @@
 	.endif
 
 	/*
+	 * Save the x0/x7/x8 in the Callee-saved registers.
+	 * We will use them in the syscall path.
+	 */
+	.if	\el == 0
+	mov	x19, x0
+	mov	x20, x7
+	mov	scno, x8
+	.endif
+
+	/*
 	 * Registers that may be useful after this macro is invoked:
 	 *
 	 * x21 - aborted SP
@@ -500,7 +510,7 @@ el0_svc_compat:
 	 * AArch32 syscall handling
 	 */
 	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
-	uxtw	scno, w7			// syscall number in w7 (r7)
+	uxtw	scno, w20			// syscall number in w7 (r7)
 	mov     sc_nr, #__NR_compat_syscalls
 	b	el0_svc_naked
 
@@ -712,10 +722,10 @@ ENDPROC(ret_from_fork)
 	.align	6
 el0_svc:
 	adrp	stbl, sys_call_table		// load syscall table pointer
-	uxtw	scno, w8			// syscall number in w8
+	uxtw	scno, w26			// syscall number in w8
 	mov	sc_nr, #__NR_syscalls
 el0_svc_naked:					// compat entry point
-	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
+	stp	x19, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
 	enable_dbg_and_irq
 	ct_user_exit 1
 
-- 
2.5.5

[PATCH 2/9] arm64: entry: add a new macro to restore the registers for syscall

[Huang Shijie]

This patch adds restore_syscall_regs to restore the x0~x7 arguments from
the stack.
And this patch also uses this macro to simplify the code.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 7dcfd49..21b6068 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -32,6 +32,17 @@
 #include <asm/unistd.h>
 
 /*
+ * Use this macro to restore the syscall arguments from the
+ * values already saved on the stack during kernel_entry.
+ */
+	.macro restore_syscall_regs
+	ldp	x0, x1, [sp]
+	ldp	x2, x3, [sp, #S_X2]
+	ldp	x4, x5, [sp, #S_X4]
+	ldp	x6, x7, [sp, #S_X6]
+	.endm
+
+/*
  * Context tracking subsystem.  Used to instrument transitions
  * between user and kernel mode.
  */
@@ -39,14 +50,7 @@
 #ifdef CONFIG_CONTEXT_TRACKING
 	bl	context_tracking_user_exit
 	.if \syscall == 1
-	/*
-	 * Save/restore needed during syscalls.  Restore syscall arguments from
-	 * the values already saved on stack during kernel_entry.
-	 */
-	ldp	x0, x1, [sp]
-	ldp	x2, x3, [sp, #S_X2]
-	ldp	x4, x5, [sp, #S_X4]
-	ldp	x6, x7, [sp, #S_X6]
+	restore_syscall_regs
 	.endif
 #endif
 	.endm
@@ -761,10 +765,7 @@ __sys_trace:
 	mov	x1, sp				// pointer to regs
 	cmp	scno, sc_nr			// check upper syscall limit
 	b.hs	__ni_sys_trace
-	ldp	x0, x1, [sp]			// restore the syscall args
-	ldp	x2, x3, [sp, #S_X2]
-	ldp	x4, x5, [sp, #S_X4]
-	ldp	x6, x7, [sp, #S_X6]
+	restore_syscall_regs
 	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
 	blr	x16				// call sys_* routine
 
-- 
2.5.5

[PATCH 3/9] arm64: entry: record the hardirq changes on the entry/exit code

[Huang Shijie]

Just as the arm32 code does, record the hardirq changes at the entry/exit
code.

The trace code may changes some registers, but the syscall path
needs the x0 ~ x7, we restore them in the el0_svc_naked.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 21b6068..4c67c56 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -132,6 +132,15 @@
 	mov	scno, x8
 	.endif
 
+#ifdef CONFIG_TRACE_IRQFLAGS
+	/*
+	 * The trace_hardirqs_off may changes some registers, so we should
+	 * restore them back from the stack in the necessary place, such as
+	 * el0_svc_naked.
+	 */
+	bl	trace_hardirqs_off
+#endif
+
 	/*
 	 * Registers that may be useful after this macro is invoked:
 	 *
@@ -142,6 +151,9 @@
 	.endm
 
 	.macro	kernel_exit, el
+#ifdef CONFIG_TRACE_IRQFLAGS
+	bl	trace_hardirqs_on
+#endif
 	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
 	.if	\el == 0
 	ct_user_enter
@@ -729,6 +741,9 @@ el0_svc:
 	uxtw	scno, w26			// syscall number in w8
 	mov	sc_nr, #__NR_syscalls
 el0_svc_naked:					// compat entry point
+#ifdef CONFIG_TRACE_IRQFLAGS
+	restore_syscall_regs
+#endif
 	stp	x19, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
 	enable_dbg_and_irq
 	ct_user_exit 1
-- 
2.5.5

[PATCH 4/9] arm64: entry: remove the duplicated code for tracing the IRQ flags

[Huang Shijie]

We record the hardirq flags changes in the entry/exit code now,
So the code becomes duplicated, just remove it.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 12 ------------
 arch/arm64/mm/fault.c     | 10 ----------
 2 files changed, 22 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 4c67c56..102fd8b 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -428,9 +428,6 @@ ENDPROC(el1_sync)
 el1_irq:
 	kernel_entry 1
 	enable_dbg
-#ifdef CONFIG_TRACE_IRQFLAGS
-	bl	trace_hardirqs_off
-#endif
 
 	get_thread_info tsk
 	irq_handler
@@ -443,9 +440,6 @@ el1_irq:
 	bl	el1_preempt
 1:
 #endif
-#ifdef CONFIG_TRACE_IRQFLAGS
-	bl	trace_hardirqs_on
-#endif
 	kernel_exit 1
 ENDPROC(el1_irq)
 
@@ -632,16 +626,10 @@ el0_irq:
 	kernel_entry 0
 el0_irq_naked:
 	enable_dbg
-#ifdef CONFIG_TRACE_IRQFLAGS
-	bl	trace_hardirqs_off
-#endif
 
 	ct_user_exit
 	irq_handler
 
-#ifdef CONFIG_TRACE_IRQFLAGS
-	bl	trace_hardirqs_on
-#endif
 	b	ret_to_user
 ENDPROC(el0_irq)
 
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 5954881..f94e598 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -603,13 +603,6 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
 	struct siginfo info;
 	int rv;
 
-	/*
-	 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were
-	 * already disabled to preserve the last enabled/disabled addresses.
-	 */
-	if (interrupts_enabled(regs))
-		trace_hardirqs_off();
-
 	if (!inf->fn(addr, esr, regs)) {
 		rv = 1;
 	} else {
@@ -624,9 +617,6 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
 		rv = 0;
 	}
 
-	if (interrupts_enabled(regs))
-		trace_hardirqs_on();
-
 	return rv;
 }
 
-- 
2.5.5

[PATCH 5/9] arm64: entry: only restore the syscall registers once

[Huang Shijie]

There are two pachs which will restore the syscall registers (x0 - x7)
separately: the hardirq trace code and the context tracking code.

This patch makes the code to restore the syscall registers only once.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 102fd8b..98d8dd1 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -46,12 +46,9 @@
  * Context tracking subsystem.  Used to instrument transitions
  * between user and kernel mode.
  */
-	.macro ct_user_exit, syscall = 0
+	.macro ct_user_exit
 #ifdef CONFIG_CONTEXT_TRACKING
 	bl	context_tracking_user_exit
-	.if \syscall == 1
-	restore_syscall_regs
-	.endif
 #endif
 	.endm
 
@@ -729,12 +726,12 @@ el0_svc:
 	uxtw	scno, w26			// syscall number in w8
 	mov	sc_nr, #__NR_syscalls
 el0_svc_naked:					// compat entry point
-#ifdef CONFIG_TRACE_IRQFLAGS
-	restore_syscall_regs
-#endif
 	stp	x19, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
 	enable_dbg_and_irq
-	ct_user_exit 1
+	ct_user_exit
+#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
+	restore_syscall_regs
+#endif
 
 	ldr	x16, [tsk, #TI_FLAGS]		// check for syscall hooks
 	tst	x16, #_TIF_SYSCALL_WORK
-- 
2.5.5

[PATCH 6/9] arm64: entry: adjust el1_sync so that a function can be called

[Huang Shijie]

To implement the hardirq flag tracing properly on arm64,
the trace_hardirqs_on() should be called before interrupts are turned on.
But the trace_hardirqs_on() may clobber the registers, such as x0 ~ 18.

Just as the el0_sync does, this patch uses the callee-saved registers:
   1.) uses x25 to save the esr_el1,
   2.) uses the x26 to save the far_el1.

And copy x25/x26 back to x1/x0 in the proper places, such as
el1_da/el1_dbg/el1_sp_pc.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 98d8dd1..63bf7ad 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -355,8 +355,8 @@ ENDPROC(el1_error_invalid)
 	.align	6
 el1_sync:
 	kernel_entry 1
-	mrs	x1, esr_el1			// read the syndrome register
-	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
+	mrs	x25, esr_el1			// read the syndrome register
+	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
 	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
 	b.eq	el1_da
 	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
@@ -374,12 +374,14 @@ el1_da:
 	/*
 	 * Data abort handling
 	 */
-	mrs	x0, far_el1
+	mrs	x26, far_el1
 	enable_dbg
 	// re-enable interrupts if they were enabled in the aborted context
 	tbnz	x23, #7, 1f			// PSR_I_BIT
 	enable_irq
 1:
+	mov	x0, x26
+	mov	x1, x25
 	mov	x2, sp				// struct pt_regs
 	bl	do_mem_abort
 
@@ -392,6 +394,7 @@ el1_sp_pc:
 	 */
 	mrs	x0, far_el1
 	enable_dbg
+	mov	x1, x25
 	mov	x2, sp
 	b	do_sp_pc_abort
 el1_undef:
@@ -409,6 +412,7 @@ el1_dbg:
 	cinc	x24, x24, eq			// set bit '0'
 	tbz	x24, #0, el1_inv		// EL1 only
 	mrs	x0, far_el1
+	mov	x1, x25
 	mov	x2, sp				// struct pt_regs
 	bl	do_debug_exception
 	kernel_exit 1
-- 
2.5.5

[PATCH 7/9] arm64: entry: save the x0 back into the stack before disabling the interrupt

[Huang Shijie]

We will add the hardirq flags trace code in the disable_irq, the trace
code may changes the x0, so save the x0 back into the stack before
disabling the interrupt,

This patch makes preparation for the later patch.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 63bf7ad..7005789 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -672,8 +672,8 @@ ENDPROC(cpu_switch_to)
  * and this includes saving x0 back into the kernel stack.
  */
 ret_fast_syscall:
-	disable_irq				// disable interrupts
 	str	x0, [sp, #S_X0]			// returned x0
+	disable_irq				// disable interrupts
 	ldr	x1, [tsk, #TI_FLAGS]		// re-check for syscall tracing
 	and	x2, x1, #_TIF_SYSCALL_WORK
 	cbnz	x2, ret_fast_syscall_trace
-- 
2.5.5

[PATCH 8/9] arm64: add the hardirq flags trace code for irq macros

[Huang Shijie]

This patch adds the hardirq flags trace code for irq macros.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/include/asm/assembler.h | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 10b017c..0be1d29 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -34,9 +34,15 @@
  */
 	.macro	disable_irq
 	msr	daifset, #2
+#ifdef CONFIG_TRACE_IRQFLAGS
+	bl	trace_hardirqs_off
+#endif
 	.endm
 
 	.macro	enable_irq
+#ifdef CONFIG_TRACE_IRQFLAGS
+	bl	trace_hardirqs_on
+#endif
 	msr	daifclr, #2
 	.endm
 
@@ -75,6 +81,9 @@
  * are self-synchronising.
  */
 	.macro	enable_dbg_and_irq
+#ifdef CONFIG_TRACE_IRQFLAGS
+	bl	trace_hardirqs_on
+#endif
 	msr	daifclr, #(8 | 2)
 	.endm
 
-- 
2.5.5

[PATCH 9/9] arm64: entry: remove the duplicated hardirq flags trace code

[Huang Shijie]

We have added the trace code to the interrupt enable/disable macros,
so the trace code in work_pending and do_notify_resume are
duplicated, just remove them.

Signed-off-by: Huang Shijie <shijie.huang@arm.com>
---
 arch/arm64/kernel/entry.S  | 3 ---
 arch/arm64/kernel/signal.c | 6 ------
 2 files changed, 9 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 7005789..b6edb55 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -691,9 +691,6 @@ ret_fast_syscall_trace:
 work_pending:
 	mov	x0, sp				// 'regs'
 	bl	do_notify_resume
-#ifdef CONFIG_TRACE_IRQFLAGS
-	bl	trace_hardirqs_on		// enabled while in userspace
-#endif
 	ldr	x1, [tsk, #TI_FLAGS]		// re-check for single-step
 	b	finish_ret_to_user
 /*
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 404dd67..56fdcb2 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -402,12 +402,6 @@ static void do_signal(struct pt_regs *regs)
 asmlinkage void do_notify_resume(struct pt_regs *regs,
 				 unsigned int thread_flags)
 {
-	/*
-	 * The assembly code enters us with IRQs off, but it hasn't
-	 * informed the tracing code of that for efficiency reasons.
-	 * Update the trace code with the current status.
-	 */
-	trace_hardirqs_off();
 	do {
 		if (thread_flags & _TIF_NEED_RESCHED) {
 			schedule();
-- 
2.5.5