* [PATCH 01/18] powerpc/64s: move the last of the page fault handling logic to C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 21:54 ` kernel test robot
2020-11-05 14:34 ` [PATCH 02/18] powerpc: remove arguments from fault handler functions Nicholas Piggin
` (16 subsequent siblings)
17 siblings, 1 reply; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
The page fault handling still has some complex logic particularly around
hash table handling, in asm. Implement this in C instead.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/bug.h | 1 +
arch/powerpc/kernel/exceptions-64s.S | 131 +++++---------------------
arch/powerpc/mm/book3s64/hash_utils.c | 77 +++++++++------
arch/powerpc/mm/fault.c | 57 ++++++++++-
4 files changed, 126 insertions(+), 140 deletions(-)
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index 338f36cd9934..d714d83bbc7c 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -112,6 +112,7 @@
struct pt_regs;
extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long);
+extern int hash__do_page_fault(struct pt_regs *, unsigned long, unsigned long);
extern void bad_page_fault(struct pt_regs *, unsigned long, int);
extern void _exception(int, struct pt_regs *, int, unsigned long);
extern void _exception_pkey(struct pt_regs *, unsigned long, int);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index f7d748b88705..f830b893fe03 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1403,14 +1403,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
*
* Handling:
* - Hash MMU
- * Go to do_hash_page first to see if the HPT can be filled from an entry in
- * the Linux page table. Hash faults can hit in kernel mode in a fairly
+ * Go to do_hash_fault, which attempts to fill the HPT from an entry in the
+ * Linux page table. Hash faults can hit in kernel mode in a fairly
* arbitrary state (e.g., interrupts disabled, locks held) when accessing
* "non-bolted" regions, e.g., vmalloc space. However these should always be
- * backed by Linux page tables.
+ * backed by Linux page table entries.
*
- * If none is found, do a Linux page fault. Linux page faults can happen in
- * kernel mode due to user copy operations of course.
+ * If no entry is found the Linux page fault handler is invoked (by
+ * do_hash_fault). Linux page faults can happen in kernel mode due to user
+ * copy operations of course.
*
* - Radix MMU
* The hardware loads from the Linux page table directly, so a fault goes
@@ -1438,13 +1439,17 @@ EXC_COMMON_BEGIN(data_access_common)
GEN_COMMON data_access
ld r4,_DAR(r1)
ld r5,_DSISR(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
- ld r6,_MSR(r1)
- li r3,0x300
- b do_hash_page /* Try to handle as hpte fault */
+ bl do_hash_fault
MMU_FTR_SECTION_ELSE
- b handle_page_fault
+ bl do_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
+ cmpdi r3,0
+ beq+ interrupt_return
+ /* We need to restore NVGPRS */
+ REST_NVGPRS(r1)
+ b interrupt_return
GEN_KVM data_access
@@ -1539,13 +1544,17 @@ EXC_COMMON_BEGIN(instruction_access_common)
GEN_COMMON instruction_access
ld r4,_DAR(r1)
ld r5,_DSISR(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
- ld r6,_MSR(r1)
- li r3,0x400
- b do_hash_page /* Try to handle as hpte fault */
+ bl do_hash_fault
MMU_FTR_SECTION_ELSE
- b handle_page_fault
+ bl do_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
+ cmpdi r3,0
+ beq+ interrupt_return
+ /* We need to restore NVGPRS */
+ REST_NVGPRS(r1)
+ b interrupt_return
GEN_KVM instruction_access
@@ -3197,99 +3206,3 @@ disable_machine_check:
RFI_TO_KERNEL
1: mtlr r0
blr
-
-/*
- * Hash table stuff
- */
- .balign IFETCH_ALIGN_BYTES
-do_hash_page:
-#ifdef CONFIG_PPC_BOOK3S_64
- lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h
- ori r0,r0,DSISR_BAD_FAULT_64S@l
- and. r0,r5,r0 /* weird error? */
- bne- handle_page_fault /* if not, try to insert a HPTE */
-
- /*
- * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
- * don't call hash_page, just fail the fault. This is required to
- * prevent re-entrancy problems in the hash code, namely perf
- * interrupts hitting while something holds H_PAGE_BUSY, and taking a
- * hash fault. See the comment in hash_preload().
- */
- ld r11, PACA_THREAD_INFO(r13)
- lwz r0,TI_PREEMPT(r11)
- andis. r0,r0,NMI_MASK@h
- bne 77f
-
- /*
- * r3 contains the trap number
- * r4 contains the faulting address
- * r5 contains dsisr
- * r6 msr
- *
- * at return r3 = 0 for success, 1 for page fault, negative for error
- */
- bl __hash_page /* build HPTE if possible */
- cmpdi r3,0 /* see if __hash_page succeeded */
-
- /* Success */
- beq interrupt_return /* Return from exception on success */
-
- /* Error */
- blt- 13f
-
- /* Reload DAR/DSISR into r4/r5 for the DABR check below */
- ld r4,_DAR(r1)
- ld r5,_DSISR(r1)
-#endif /* CONFIG_PPC_BOOK3S_64 */
-
-/* Here we have a page fault that hash_page can't handle. */
-handle_page_fault:
-11: andis. r0,r5,DSISR_DABRMATCH@h
- bne- handle_dabr_fault
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl do_page_fault
- cmpdi r3,0
- beq+ interrupt_return
- mr r5,r3
- addi r3,r1,STACK_FRAME_OVERHEAD
- ld r4,_DAR(r1)
- bl bad_page_fault
- b interrupt_return
-
-/* We have a data breakpoint exception - handle it */
-handle_dabr_fault:
- ld r4,_DAR(r1)
- ld r5,_DSISR(r1)
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl do_break
- /*
- * do_break() may have changed the NV GPRS while handling a breakpoint.
- * If so, we need to restore them with their updated values.
- */
- REST_NVGPRS(r1)
- b interrupt_return
-
-
-#ifdef CONFIG_PPC_BOOK3S_64
-/* We have a page fault that hash_page could handle but HV refused
- * the PTE insertion
- */
-13: mr r5,r3
- addi r3,r1,STACK_FRAME_OVERHEAD
- ld r4,_DAR(r1)
- bl low_hash_fault
- b interrupt_return
-#endif
-
-/*
- * We come here as a result of a DSI at a point where we don't want
- * to call hash_page, such as when we are accessing memory (possibly
- * user memory) inside a PMU interrupt that occurred while interrupts
- * were soft-disabled. We want to invoke the exception handler for
- * the access, or panic if there isn't a handler.
- */
-77: addi r3,r1,STACK_FRAME_OVERHEAD
- li r5,SIGSEGV
- bl bad_page_fault
- b interrupt_return
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index 24702c0a92e0..bfa1b1966218 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -1510,16 +1510,40 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
}
EXPORT_SYMBOL_GPL(hash_page);
-int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr,
- unsigned long msr)
+int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
{
unsigned long access = _PAGE_PRESENT | _PAGE_READ;
unsigned long flags = 0;
- struct mm_struct *mm = current->mm;
- unsigned int region_id = get_region_id(ea);
+ struct mm_struct *mm;
+ unsigned int region_id;
+ int err;
+
+ if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)))
+ goto _do_page_fault;
+
+ /*
+ * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
+ * don't call hash_page, just fail the fault. This is required to
+ * prevent re-entrancy problems in the hash code, namely perf
+ * interrupts hitting while something holds H_PAGE_BUSY, and taking a
+ * hash fault. See the comment in hash_preload().
+ *
+ * We come here as a result of a DSI at a point where we don't want
+ * to call hash_page, such as when we are accessing memory (possibly
+ * user memory) inside a PMU interrupt that occurred while interrupts
+ * were soft-disabled. We want to invoke the exception handler for
+ * the access, or panic if there isn't a handler.
+ */
+ if (unlikely(in_nmi())) {
+ bad_page_fault(regs, ea, SIGSEGV);
+ return 0;
+ }
+ region_id = get_region_id(ea);
if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID))
mm = &init_mm;
+ else
+ mm = current->mm;
if (dsisr & DSISR_NOHPTE)
flags |= HPTE_NOHPTE_UPDATE;
@@ -1535,13 +1559,31 @@ int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr,
* 2) user space access kernel space.
*/
access |= _PAGE_PRIVILEGED;
- if ((msr & MSR_PR) || (region_id == USER_REGION_ID))
+ if (user_mode(regs) || (region_id == USER_REGION_ID))
access &= ~_PAGE_PRIVILEGED;
- if (trap == 0x400)
+ if (regs->trap == 0x400)
access |= _PAGE_EXEC;
- return hash_page_mm(mm, ea, access, trap, flags);
+ err = hash_page_mm(mm, ea, access, regs->trap, flags);
+ if (unlikely(err < 0)) {
+ // failed to instert a hash PTE due to an hypervisor error
+ if (user_mode(regs)) {
+ if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2)
+ _exception(SIGSEGV, regs, SEGV_ACCERR, ea);
+ else
+ _exception(SIGBUS, regs, BUS_ADRERR, ea);
+ } else {
+ bad_page_fault(regs, ea, SIGBUS);
+ }
+ err = 0;
+
+ } else if (err) {
+_do_page_fault:
+ err = hash__do_page_fault(regs, ea, dsisr);
+ }
+
+ return err;
}
#ifdef CONFIG_PPC_MM_SLICES
@@ -1841,27 +1883,6 @@ void flush_hash_range(unsigned long number, int local)
}
}
-/*
- * low_hash_fault is called when we the low level hash code failed
- * to instert a PTE due to an hypervisor error
- */
-void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
-{
- enum ctx_state prev_state = exception_enter();
-
- if (user_mode(regs)) {
-#ifdef CONFIG_PPC_SUBPAGE_PROT
- if (rc == -2)
- _exception(SIGSEGV, regs, SEGV_ACCERR, address);
- else
-#endif
- _exception(SIGBUS, regs, BUS_ADRERR, address);
- } else
- bad_page_fault(regs, address, SIGBUS);
-
- exception_exit(prev_state);
-}
-
long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
unsigned long pa, unsigned long rflags,
unsigned long vflags, int psize, int ssize)
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 0add963a849b..e65a49f246ef 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -370,7 +370,9 @@ static void sanity_check_fault(bool is_write, bool is_user,
#define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE)
#if defined(CONFIG_PPC_8xx)
#define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G)
-#elif defined(CONFIG_PPC64)
+#elif defined(CONFIG_PPC_BOOK3S_64)
+#define page_fault_is_bad(__err) ((__err) & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH))
+#elif defined(CONFIG_PPC_BOOK3E_64)
#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S)
#else
#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S)
@@ -406,6 +408,9 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
return 0;
if (unlikely(page_fault_is_bad(error_code))) {
+ if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && (error_code & DSISR_DABRMATCH))
+ return -1;
+
if (is_user) {
_exception(SIGBUS, regs, BUS_OBJERR, address);
return 0;
@@ -548,12 +553,58 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
unsigned long error_code)
{
enum ctx_state prev_state = exception_enter();
- int rc = __do_page_fault(regs, address, error_code);
+ int err;
+
+ err = __do_page_fault(regs, address, error_code);
+
+#ifdef CONFIG_PPC_BOOK3S_64
+ /* 32 and 64e handle errors in their asm code */
+ if (unlikely(err)) {
+ if (err > 0) {
+ bad_page_fault(regs, address, err);
+ err = 0;
+ } else {
+ /*
+ * do_break() may change NV GPRS while handling the
+ * breakpoint. Return -ve to caller to do that.
+ */
+ do_break(regs, address, error_code);
+ }
+ }
+#endif
+
exception_exit(prev_state);
- return rc;
+
+ return err;
}
NOKPROBE_SYMBOL(do_page_fault);
+#ifdef CONFIG_PPC_BOOK3S_64
+/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */
+int hash__do_page_fault(struct pt_regs *regs, unsigned long address,
+ unsigned long error_code)
+{
+ int err;
+
+ err = __do_page_fault(regs, address, error_code);
+ if (unlikely(err)) {
+ if (err > 0) {
+ bad_page_fault(regs, address, err);
+ err = 0;
+ } else {
+ /*
+ * do_break() may change NV GPRS while handling the
+ * breakpoint. Return -ve to caller to do that.
+ */
+ do_break(regs, address, error_code);
+ }
+ }
+
+ return err;
+}
+NOKPROBE_SYMBOL(hash__do_page_fault);
+#endif
+
/*
* bad_page_fault is called when we have a bad access from the kernel.
* It is called from the DSI and ISI handlers in head.S and from some
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH 01/18] powerpc/64s: move the last of the page fault handling logic to C
2020-11-05 14:34 ` [PATCH 01/18] powerpc/64s: move the last of the page fault handling logic to C Nicholas Piggin
@ 2020-11-05 21:54 ` kernel test robot
0 siblings, 0 replies; 36+ messages in thread
From: kernel test robot @ 2020-11-05 21:54 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev; +Cc: kbuild-all, Nicholas Piggin
[-- Attachment #1: Type: text/plain, Size: 4810 bytes --]
Hi Nicholas,
I love your patch! Perhaps something to improve:
[auto build test WARNING on powerpc/next]
[also build test WARNING on v5.10-rc2 next-20201105]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-allyesconfig (attached as .config)
compiler: powerpc64-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/7c4e3c4d325c8c43a43b8031b5327cb91b56db19
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
git checkout 7c4e3c4d325c8c43a43b8031b5327cb91b56db19
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
>> arch/powerpc/mm/book3s64/hash_utils.c:1513:5: warning: no previous prototype for 'do_hash_fault' [-Wmissing-prototypes]
1513 | int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
| ^~~~~~~~~~~~~
arch/powerpc/mm/book3s64/hash_utils.c:1886:6: warning: no previous prototype for 'hpte_insert_repeating' [-Wmissing-prototypes]
1886 | long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
| ^~~~~~~~~~~~~~~~~~~~~
vim +/do_hash_fault +1513 arch/powerpc/mm/book3s64/hash_utils.c
1512
> 1513 int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
1514 {
1515 unsigned long access = _PAGE_PRESENT | _PAGE_READ;
1516 unsigned long flags = 0;
1517 struct mm_struct *mm;
1518 unsigned int region_id;
1519 int err;
1520
1521 if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)))
1522 goto _do_page_fault;
1523
1524 /*
1525 * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
1526 * don't call hash_page, just fail the fault. This is required to
1527 * prevent re-entrancy problems in the hash code, namely perf
1528 * interrupts hitting while something holds H_PAGE_BUSY, and taking a
1529 * hash fault. See the comment in hash_preload().
1530 *
1531 * We come here as a result of a DSI at a point where we don't want
1532 * to call hash_page, such as when we are accessing memory (possibly
1533 * user memory) inside a PMU interrupt that occurred while interrupts
1534 * were soft-disabled. We want to invoke the exception handler for
1535 * the access, or panic if there isn't a handler.
1536 */
1537 if (unlikely(in_nmi())) {
1538 bad_page_fault(regs, ea, SIGSEGV);
1539 return 0;
1540 }
1541
1542 region_id = get_region_id(ea);
1543 if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID))
1544 mm = &init_mm;
1545 else
1546 mm = current->mm;
1547
1548 if (dsisr & DSISR_NOHPTE)
1549 flags |= HPTE_NOHPTE_UPDATE;
1550
1551 if (dsisr & DSISR_ISSTORE)
1552 access |= _PAGE_WRITE;
1553 /*
1554 * We set _PAGE_PRIVILEGED only when
1555 * kernel mode access kernel space.
1556 *
1557 * _PAGE_PRIVILEGED is NOT set
1558 * 1) when kernel mode access user space
1559 * 2) user space access kernel space.
1560 */
1561 access |= _PAGE_PRIVILEGED;
1562 if (user_mode(regs) || (region_id == USER_REGION_ID))
1563 access &= ~_PAGE_PRIVILEGED;
1564
1565 if (regs->trap == 0x400)
1566 access |= _PAGE_EXEC;
1567
1568 err = hash_page_mm(mm, ea, access, regs->trap, flags);
1569 if (unlikely(err < 0)) {
1570 // failed to instert a hash PTE due to an hypervisor error
1571 if (user_mode(regs)) {
1572 if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2)
1573 _exception(SIGSEGV, regs, SEGV_ACCERR, ea);
1574 else
1575 _exception(SIGBUS, regs, BUS_ADRERR, ea);
1576 } else {
1577 bad_page_fault(regs, ea, SIGBUS);
1578 }
1579 err = 0;
1580
1581 } else if (err) {
1582 _do_page_fault:
1583 err = hash__do_page_fault(regs, ea, dsisr);
1584 }
1585
1586 return err;
1587 }
1588
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 71468 bytes --]
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 01/18] powerpc/64s: move the last of the page fault handling logic to C
@ 2020-11-05 21:54 ` kernel test robot
0 siblings, 0 replies; 36+ messages in thread
From: kernel test robot @ 2020-11-05 21:54 UTC (permalink / raw)
To: kbuild-all
[-- Attachment #1: Type: text/plain, Size: 4929 bytes --]
Hi Nicholas,
I love your patch! Perhaps something to improve:
[auto build test WARNING on powerpc/next]
[also build test WARNING on v5.10-rc2 next-20201105]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-allyesconfig (attached as .config)
compiler: powerpc64-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/7c4e3c4d325c8c43a43b8031b5327cb91b56db19
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
git checkout 7c4e3c4d325c8c43a43b8031b5327cb91b56db19
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
>> arch/powerpc/mm/book3s64/hash_utils.c:1513:5: warning: no previous prototype for 'do_hash_fault' [-Wmissing-prototypes]
1513 | int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
| ^~~~~~~~~~~~~
arch/powerpc/mm/book3s64/hash_utils.c:1886:6: warning: no previous prototype for 'hpte_insert_repeating' [-Wmissing-prototypes]
1886 | long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
| ^~~~~~~~~~~~~~~~~~~~~
vim +/do_hash_fault +1513 arch/powerpc/mm/book3s64/hash_utils.c
1512
> 1513 int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
1514 {
1515 unsigned long access = _PAGE_PRESENT | _PAGE_READ;
1516 unsigned long flags = 0;
1517 struct mm_struct *mm;
1518 unsigned int region_id;
1519 int err;
1520
1521 if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)))
1522 goto _do_page_fault;
1523
1524 /*
1525 * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
1526 * don't call hash_page, just fail the fault. This is required to
1527 * prevent re-entrancy problems in the hash code, namely perf
1528 * interrupts hitting while something holds H_PAGE_BUSY, and taking a
1529 * hash fault. See the comment in hash_preload().
1530 *
1531 * We come here as a result of a DSI at a point where we don't want
1532 * to call hash_page, such as when we are accessing memory (possibly
1533 * user memory) inside a PMU interrupt that occurred while interrupts
1534 * were soft-disabled. We want to invoke the exception handler for
1535 * the access, or panic if there isn't a handler.
1536 */
1537 if (unlikely(in_nmi())) {
1538 bad_page_fault(regs, ea, SIGSEGV);
1539 return 0;
1540 }
1541
1542 region_id = get_region_id(ea);
1543 if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID))
1544 mm = &init_mm;
1545 else
1546 mm = current->mm;
1547
1548 if (dsisr & DSISR_NOHPTE)
1549 flags |= HPTE_NOHPTE_UPDATE;
1550
1551 if (dsisr & DSISR_ISSTORE)
1552 access |= _PAGE_WRITE;
1553 /*
1554 * We set _PAGE_PRIVILEGED only when
1555 * kernel mode access kernel space.
1556 *
1557 * _PAGE_PRIVILEGED is NOT set
1558 * 1) when kernel mode access user space
1559 * 2) user space access kernel space.
1560 */
1561 access |= _PAGE_PRIVILEGED;
1562 if (user_mode(regs) || (region_id == USER_REGION_ID))
1563 access &= ~_PAGE_PRIVILEGED;
1564
1565 if (regs->trap == 0x400)
1566 access |= _PAGE_EXEC;
1567
1568 err = hash_page_mm(mm, ea, access, regs->trap, flags);
1569 if (unlikely(err < 0)) {
1570 // failed to instert a hash PTE due to an hypervisor error
1571 if (user_mode(regs)) {
1572 if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2)
1573 _exception(SIGSEGV, regs, SEGV_ACCERR, ea);
1574 else
1575 _exception(SIGBUS, regs, BUS_ADRERR, ea);
1576 } else {
1577 bad_page_fault(regs, ea, SIGBUS);
1578 }
1579 err = 0;
1580
1581 } else if (err) {
1582 _do_page_fault:
1583 err = hash__do_page_fault(regs, ea, dsisr);
1584 }
1585
1586 return err;
1587 }
1588
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 71468 bytes --]
^ permalink raw reply [flat|nested] 36+ messages in thread
* [PATCH 02/18] powerpc: remove arguments from fault handler functions
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
2020-11-05 14:34 ` [PATCH 01/18] powerpc/64s: move the last of the page fault handling logic to C Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-06 7:59 ` Christophe Leroy
2020-11-05 14:34 ` [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs Nicholas Piggin
` (15 subsequent siblings)
17 siblings, 1 reply; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Make mm fault handlers all just take the pt_regs * argument and load
DAR/DSISR from that. Make those that return a value return long.
This is done to make the function signatures match other handlers, which
will help with a future patch to add wrappers. Explicit arguments could
be added for performance but that would require more wrapper macro
variants.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/asm-prototypes.h | 4 ++--
arch/powerpc/include/asm/bug.h | 4 ++--
arch/powerpc/kernel/exceptions-64e.S | 2 --
arch/powerpc/kernel/exceptions-64s.S | 14 ++------------
arch/powerpc/kernel/head_40x.S | 10 +++++-----
arch/powerpc/kernel/head_8xx.S | 6 +++---
arch/powerpc/kernel/head_book3s_32.S | 6 ++----
arch/powerpc/kernel/head_booke.h | 4 +---
arch/powerpc/mm/book3s64/hash_utils.c | 8 +++++---
arch/powerpc/mm/book3s64/slb.c | 11 +++++++----
arch/powerpc/mm/fault.c | 16 +++++++++-------
11 files changed, 38 insertions(+), 47 deletions(-)
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index d0b832cbbec8..22c9d08fa3a4 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -82,8 +82,8 @@ void kernel_bad_stack(struct pt_regs *regs);
void system_reset_exception(struct pt_regs *regs);
void machine_check_exception(struct pt_regs *regs);
void emulation_assist_interrupt(struct pt_regs *regs);
-long do_slb_fault(struct pt_regs *regs, unsigned long ea);
-void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err);
+long do_slb_fault(struct pt_regs *regs);
+void do_bad_slb_fault(struct pt_regs *regs);
/* signals, syscalls and interrupts */
long sys_swapcontext(struct ucontext __user *old_ctx,
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index d714d83bbc7c..2fa0cf6c6011 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -111,8 +111,8 @@
#ifndef __ASSEMBLY__
struct pt_regs;
-extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long);
-extern int hash__do_page_fault(struct pt_regs *, unsigned long, unsigned long);
+extern long do_page_fault(struct pt_regs *);
+extern long hash__do_page_fault(struct pt_regs *);
extern void bad_page_fault(struct pt_regs *, unsigned long, int);
extern void _exception(int, struct pt_regs *, int, unsigned long);
extern void _exception_pkey(struct pt_regs *, unsigned long, int);
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index f579ce46eef2..25fa7d5a643c 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -1011,8 +1011,6 @@ storage_fault_common:
std r14,_DAR(r1)
std r15,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
- mr r4,r14
- mr r5,r15
ld r14,PACA_EXGEN+EX_R14(r13)
ld r15,PACA_EXGEN+EX_R15(r13)
bl do_page_fault
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index f830b893fe03..1f34cfd1887c 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1437,8 +1437,6 @@ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
EXC_VIRT_END(data_access, 0x4300, 0x80)
EXC_COMMON_BEGIN(data_access_common)
GEN_COMMON data_access
- ld r4,_DAR(r1)
- ld r5,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
bl do_hash_fault
@@ -1491,10 +1489,9 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
EXC_COMMON_BEGIN(data_access_slb_common)
GEN_COMMON data_access_slb
- ld r4,_DAR(r1)
- addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
/* HPT case, do SLB fault */
+ addi r3,r1,STACK_FRAME_OVERHEAD
bl do_slb_fault
cmpdi r3,0
bne- 1f
@@ -1506,8 +1503,6 @@ MMU_FTR_SECTION_ELSE
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
std r3,RESULT(r1)
RECONCILE_IRQ_STATE(r10, r11)
- ld r4,_DAR(r1)
- ld r5,RESULT(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_bad_slb_fault
b interrupt_return
@@ -1542,8 +1537,6 @@ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
EXC_VIRT_END(instruction_access, 0x4400, 0x80)
EXC_COMMON_BEGIN(instruction_access_common)
GEN_COMMON instruction_access
- ld r4,_DAR(r1)
- ld r5,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
bl do_hash_fault
@@ -1587,10 +1580,9 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
EXC_COMMON_BEGIN(instruction_access_slb_common)
GEN_COMMON instruction_access_slb
- ld r4,_DAR(r1)
- addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
/* HPT case, do SLB fault */
+ addi r3,r1,STACK_FRAME_OVERHEAD
bl do_slb_fault
cmpdi r3,0
bne- 1f
@@ -1602,8 +1594,6 @@ MMU_FTR_SECTION_ELSE
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
std r3,RESULT(r1)
RECONCILE_IRQ_STATE(r10, r11)
- ld r4,_DAR(r1)
- ld r5,RESULT(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_bad_slb_fault
b interrupt_return
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 44c9018aed1b..ea31f75e9692 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit)
*/
START_EXCEPTION(0x0300, DataStorage)
EXCEPTION_PROLOG
- mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */
+ mfspr r5, SPRN_ESR /* Grab the ESR, save it */
stw r5, _ESR(r11)
- mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */
+ mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */
stw r4, _DEAR(r11)
EXC_XFER_LITE(0x300, handle_page_fault)
@@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit)
*/
START_EXCEPTION(0x0400, InstructionAccess)
EXCEPTION_PROLOG
- mr r4,r12 /* Pass SRR0 as arg2 */
- stw r4, _DEAR(r11)
- li r5,0 /* Pass zero as arg3 */
+ li r5,0
+ stw r5, _ESR(r11) /* Zero ESR */
+ stw r12, _DEAR(r11) /* SRR0 as DEAR */
EXC_XFER_LITE(0x400, handle_page_fault)
/* 0x0500 - External Interrupt Exception */
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 9f359d3fba74..0cd95b633e2b 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -356,14 +356,14 @@ DataStoreTLBMiss:
. = 0x1300
InstructionTLBError:
EXCEPTION_PROLOG
- mr r4,r12
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
andis. r10,r9,SRR1_ISI_NOPT@h
beq+ .Litlbie
- tlbie r4
+ tlbie r12
/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
.Litlbie:
- stw r4, _DAR(r11)
+ stw r12, _DAR(r11)
+ stw r5, _DSISR(r11)
EXC_XFER_LITE(0x400, handle_page_fault)
/* This is the data TLB error on the MPC8xx. This could be due to
diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
index 5eb9eedac920..81c69769cec6 100644
--- a/arch/powerpc/kernel/head_book3s_32.S
+++ b/arch/powerpc/kernel/head_book3s_32.S
@@ -369,9 +369,9 @@ BEGIN_MMU_FTR_SECTION
bl hash_page
END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
#endif /* CONFIG_VMAP_STACK */
-1: mr r4,r12
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
- stw r4, _DAR(r11)
+ stw r5, _DSISR(r11)
+ stw r12, _DAR(r11)
EXC_XFER_LITE(0x400, handle_page_fault)
/* External interrupt */
@@ -698,8 +698,6 @@ handle_page_fault_tramp_1:
#ifdef CONFIG_VMAP_STACK
EXCEPTION_PROLOG_2 handle_dar_dsisr=1
#endif
- lwz r4, _DAR(r11)
- lwz r5, _DSISR(r11)
/* fall through */
handle_page_fault_tramp_2:
EXC_XFER_LITE(0x300, handle_page_fault)
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 71c359d438b5..1da0c1d1b0a1 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -477,9 +477,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \
mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
stw r5,_ESR(r11); \
- mr r4,r12; /* Pass SRR0 as arg2 */ \
- stw r4, _DEAR(r11); \
- li r5,0; /* Pass zero as arg3 */ \
+ stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
EXC_XFER_LITE(0x0400, handle_page_fault)
#define ALIGNMENT_EXCEPTION \
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index bfa1b1966218..0f0bd4af4b2d 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -1510,13 +1510,15 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
}
EXPORT_SYMBOL_GPL(hash_page);
-int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
+long do_hash_fault(struct pt_regs *regs)
{
+ unsigned long ea = regs->dar;
+ unsigned long dsisr = regs->dsisr;
unsigned long access = _PAGE_PRESENT | _PAGE_READ;
unsigned long flags = 0;
struct mm_struct *mm;
unsigned int region_id;
- int err;
+ long err;
if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)))
goto _do_page_fault;
@@ -1580,7 +1582,7 @@ int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
} else if (err) {
_do_page_fault:
- err = hash__do_page_fault(regs, ea, dsisr);
+ err = hash__do_page_fault(regs);
}
return err;
diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c
index c30fcbfa0e32..cc34d50874c1 100644
--- a/arch/powerpc/mm/book3s64/slb.c
+++ b/arch/powerpc/mm/book3s64/slb.c
@@ -837,8 +837,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)
return slb_insert_entry(ea, context, flags, ssize, false);
}
-long do_slb_fault(struct pt_regs *regs, unsigned long ea)
+long do_slb_fault(struct pt_regs *regs)
{
+ unsigned long ea = regs->dar;
unsigned long id = get_region_id(ea);
/* IRQs are not reconciled here, so can't check irqs_disabled */
@@ -889,13 +890,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea)
}
}
-void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err)
+void do_bad_slb_fault(struct pt_regs *regs)
{
+ int err = regs->result;
+
if (err == -EFAULT) {
if (user_mode(regs))
- _exception(SIGSEGV, regs, SEGV_BNDERR, ea);
+ _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar);
else
- bad_page_fault(regs, ea, SIGSEGV);
+ bad_page_fault(regs, regs->dar, SIGSEGV);
} else if (err == -EINVAL) {
unrecoverable_exception(regs);
} else {
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index e65a49f246ef..390a296b16a3 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
}
NOKPROBE_SYMBOL(__do_page_fault);
-int do_page_fault(struct pt_regs *regs, unsigned long address,
- unsigned long error_code)
+long do_page_fault(struct pt_regs *regs)
{
enum ctx_state prev_state = exception_enter();
- int err;
+ unsigned long address = regs->dar;
+ unsigned long error_code = regs->dsisr;
+ long err;
err = __do_page_fault(regs, address, error_code);
@@ -580,11 +581,12 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
NOKPROBE_SYMBOL(do_page_fault);
#ifdef CONFIG_PPC_BOOK3S_64
-/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */
-int hash__do_page_fault(struct pt_regs *regs, unsigned long address,
- unsigned long error_code)
+/* Same as do_page_fault but no interrupt entry */
+long hash__do_page_fault(struct pt_regs *regs)
{
- int err;
+ unsigned long address = regs->dar;
+ unsigned long error_code = regs->dsisr;
+ long err;
err = __do_page_fault(regs, address, error_code);
if (unlikely(err)) {
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH 02/18] powerpc: remove arguments from fault handler functions
2020-11-05 14:34 ` [PATCH 02/18] powerpc: remove arguments from fault handler functions Nicholas Piggin
@ 2020-11-06 7:59 ` Christophe Leroy
2020-11-10 8:29 ` Nicholas Piggin
0 siblings, 1 reply; 36+ messages in thread
From: Christophe Leroy @ 2020-11-06 7:59 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev
Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
> Make mm fault handlers all just take the pt_regs * argument and load
> DAR/DSISR from that. Make those that return a value return long.
>
> This is done to make the function signatures match other handlers, which
> will help with a future patch to add wrappers. Explicit arguments could
> be added for performance but that would require more wrapper macro
> variants.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> ---
> arch/powerpc/include/asm/asm-prototypes.h | 4 ++--
> arch/powerpc/include/asm/bug.h | 4 ++--
> arch/powerpc/kernel/exceptions-64e.S | 2 --
> arch/powerpc/kernel/exceptions-64s.S | 14 ++------------
> arch/powerpc/kernel/head_40x.S | 10 +++++-----
> arch/powerpc/kernel/head_8xx.S | 6 +++---
> arch/powerpc/kernel/head_book3s_32.S | 6 ++----
> arch/powerpc/kernel/head_booke.h | 4 +---
> arch/powerpc/mm/book3s64/hash_utils.c | 8 +++++---
> arch/powerpc/mm/book3s64/slb.c | 11 +++++++----
> arch/powerpc/mm/fault.c | 16 +++++++++-------
> 11 files changed, 38 insertions(+), 47 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
> index d0b832cbbec8..22c9d08fa3a4 100644
> --- a/arch/powerpc/include/asm/asm-prototypes.h
> +++ b/arch/powerpc/include/asm/asm-prototypes.h
> @@ -82,8 +82,8 @@ void kernel_bad_stack(struct pt_regs *regs);
> void system_reset_exception(struct pt_regs *regs);
> void machine_check_exception(struct pt_regs *regs);
> void emulation_assist_interrupt(struct pt_regs *regs);
> -long do_slb_fault(struct pt_regs *regs, unsigned long ea);
> -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err);
> +long do_slb_fault(struct pt_regs *regs);
> +void do_bad_slb_fault(struct pt_regs *regs);
>
> /* signals, syscalls and interrupts */
> long sys_swapcontext(struct ucontext __user *old_ctx,
> diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
> index d714d83bbc7c..2fa0cf6c6011 100644
> --- a/arch/powerpc/include/asm/bug.h
> +++ b/arch/powerpc/include/asm/bug.h
> @@ -111,8 +111,8 @@
> #ifndef __ASSEMBLY__
>
> struct pt_regs;
> -extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long);
> -extern int hash__do_page_fault(struct pt_regs *, unsigned long, unsigned long);
> +extern long do_page_fault(struct pt_regs *);
> +extern long hash__do_page_fault(struct pt_regs *);
extern is pointless
> extern void bad_page_fault(struct pt_regs *, unsigned long, int);
> extern void _exception(int, struct pt_regs *, int, unsigned long);
> extern void _exception_pkey(struct pt_regs *, unsigned long, int);
> diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
> index f579ce46eef2..25fa7d5a643c 100644
> --- a/arch/powerpc/kernel/exceptions-64e.S
> +++ b/arch/powerpc/kernel/exceptions-64e.S
> @@ -1011,8 +1011,6 @@ storage_fault_common:
> std r14,_DAR(r1)
> std r15,_DSISR(r1)
> addi r3,r1,STACK_FRAME_OVERHEAD
> - mr r4,r14
> - mr r5,r15
> ld r14,PACA_EXGEN+EX_R14(r13)
> ld r15,PACA_EXGEN+EX_R15(r13)
> bl do_page_fault
> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> index f830b893fe03..1f34cfd1887c 100644
> --- a/arch/powerpc/kernel/exceptions-64s.S
> +++ b/arch/powerpc/kernel/exceptions-64s.S
> @@ -1437,8 +1437,6 @@ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
> EXC_VIRT_END(data_access, 0x4300, 0x80)
> EXC_COMMON_BEGIN(data_access_common)
> GEN_COMMON data_access
> - ld r4,_DAR(r1)
> - ld r5,_DSISR(r1)
> addi r3,r1,STACK_FRAME_OVERHEAD
> BEGIN_MMU_FTR_SECTION
> bl do_hash_fault
> @@ -1491,10 +1489,9 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
> EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
> EXC_COMMON_BEGIN(data_access_slb_common)
> GEN_COMMON data_access_slb
> - ld r4,_DAR(r1)
> - addi r3,r1,STACK_FRAME_OVERHEAD
> BEGIN_MMU_FTR_SECTION
> /* HPT case, do SLB fault */
> + addi r3,r1,STACK_FRAME_OVERHEAD
> bl do_slb_fault
> cmpdi r3,0
> bne- 1f
> @@ -1506,8 +1503,6 @@ MMU_FTR_SECTION_ELSE
> ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
> std r3,RESULT(r1)
> RECONCILE_IRQ_STATE(r10, r11)
> - ld r4,_DAR(r1)
> - ld r5,RESULT(r1)
> addi r3,r1,STACK_FRAME_OVERHEAD
> bl do_bad_slb_fault
> b interrupt_return
> @@ -1542,8 +1537,6 @@ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
> EXC_VIRT_END(instruction_access, 0x4400, 0x80)
> EXC_COMMON_BEGIN(instruction_access_common)
> GEN_COMMON instruction_access
> - ld r4,_DAR(r1)
> - ld r5,_DSISR(r1)
> addi r3,r1,STACK_FRAME_OVERHEAD
> BEGIN_MMU_FTR_SECTION
> bl do_hash_fault
> @@ -1587,10 +1580,9 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
> EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
> EXC_COMMON_BEGIN(instruction_access_slb_common)
> GEN_COMMON instruction_access_slb
> - ld r4,_DAR(r1)
> - addi r3,r1,STACK_FRAME_OVERHEAD
> BEGIN_MMU_FTR_SECTION
> /* HPT case, do SLB fault */
> + addi r3,r1,STACK_FRAME_OVERHEAD
> bl do_slb_fault
> cmpdi r3,0
> bne- 1f
> @@ -1602,8 +1594,6 @@ MMU_FTR_SECTION_ELSE
> ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
> std r3,RESULT(r1)
> RECONCILE_IRQ_STATE(r10, r11)
> - ld r4,_DAR(r1)
> - ld r5,RESULT(r1)
> addi r3,r1,STACK_FRAME_OVERHEAD
> bl do_bad_slb_fault
> b interrupt_return
> diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
> index 44c9018aed1b..ea31f75e9692 100644
> --- a/arch/powerpc/kernel/head_40x.S
> +++ b/arch/powerpc/kernel/head_40x.S
> @@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit)
> */
> START_EXCEPTION(0x0300, DataStorage)
> EXCEPTION_PROLOG
> - mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */
> + mfspr r5, SPRN_ESR /* Grab the ESR, save it */
> stw r5, _ESR(r11)
> - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */
> + mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */
> stw r4, _DEAR(r11)
> EXC_XFER_LITE(0x300, handle_page_fault)
>
> @@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit)
> */
> START_EXCEPTION(0x0400, InstructionAccess)
> EXCEPTION_PROLOG
> - mr r4,r12 /* Pass SRR0 as arg2 */
> - stw r4, _DEAR(r11)
> - li r5,0 /* Pass zero as arg3 */
> + li r5,0
> + stw r5, _ESR(r11) /* Zero ESR */
> + stw r12, _DEAR(r11) /* SRR0 as DEAR */
I think we should avoid this, see below
> EXC_XFER_LITE(0x400, handle_page_fault)
>
> /* 0x0500 - External Interrupt Exception */
> diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
> index 9f359d3fba74..0cd95b633e2b 100644
> --- a/arch/powerpc/kernel/head_8xx.S
> +++ b/arch/powerpc/kernel/head_8xx.S
> @@ -356,14 +356,14 @@ DataStoreTLBMiss:
> . = 0x1300
> InstructionTLBError:
> EXCEPTION_PROLOG
> - mr r4,r12
> andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
Could avoid this, see below
> andis. r10,r9,SRR1_ISI_NOPT@h
> beq+ .Litlbie
> - tlbie r4
> + tlbie r12
> /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
> .Litlbie:
> - stw r4, _DAR(r11)
> + stw r12, _DAR(r11)
> + stw r5, _DSISR(r11)
And this
> EXC_XFER_LITE(0x400, handle_page_fault)
>
> /* This is the data TLB error on the MPC8xx. This could be due to
> diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
> index 5eb9eedac920..81c69769cec6 100644
> --- a/arch/powerpc/kernel/head_book3s_32.S
> +++ b/arch/powerpc/kernel/head_book3s_32.S
> @@ -369,9 +369,9 @@ BEGIN_MMU_FTR_SECTION
> bl hash_page
> END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
> #endif /* CONFIG_VMAP_STACK */
> -1: mr r4,r12
> andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
> - stw r4, _DAR(r11)
> + stw r5, _DSISR(r11)
> + stw r12, _DAR(r11)
And this including the andis.
> EXC_XFER_LITE(0x400, handle_page_fault)
>
> /* External interrupt */
> @@ -698,8 +698,6 @@ handle_page_fault_tramp_1:
> #ifdef CONFIG_VMAP_STACK
> EXCEPTION_PROLOG_2 handle_dar_dsisr=1
> #endif
> - lwz r4, _DAR(r11)
> - lwz r5, _DSISR(r11)
> /* fall through */
> handle_page_fault_tramp_2:
> EXC_XFER_LITE(0x300, handle_page_fault)
> diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
> index 71c359d438b5..1da0c1d1b0a1 100644
> --- a/arch/powerpc/kernel/head_booke.h
> +++ b/arch/powerpc/kernel/head_booke.h
> @@ -477,9 +477,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
> NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \
> mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
> stw r5,_ESR(r11); \
> - mr r4,r12; /* Pass SRR0 as arg2 */ \
> - stw r4, _DEAR(r11); \
> - li r5,0; /* Pass zero as arg3 */ \
> + stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
And this
> EXC_XFER_LITE(0x0400, handle_page_fault)
>
> #define ALIGNMENT_EXCEPTION \
> diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
> index bfa1b1966218..0f0bd4af4b2d 100644
> --- a/arch/powerpc/mm/book3s64/hash_utils.c
> +++ b/arch/powerpc/mm/book3s64/hash_utils.c
> @@ -1510,13 +1510,15 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
> }
> EXPORT_SYMBOL_GPL(hash_page);
>
> -int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
> +long do_hash_fault(struct pt_regs *regs)
> {
> + unsigned long ea = regs->dar;
> + unsigned long dsisr = regs->dsisr;
> unsigned long access = _PAGE_PRESENT | _PAGE_READ;
> unsigned long flags = 0;
> struct mm_struct *mm;
> unsigned int region_id;
> - int err;
> + long err;
>
> if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)))
> goto _do_page_fault;
> @@ -1580,7 +1582,7 @@ int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
>
> } else if (err) {
> _do_page_fault:
> - err = hash__do_page_fault(regs, ea, dsisr);
> + err = hash__do_page_fault(regs);
> }
>
> return err;
> diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c
> index c30fcbfa0e32..cc34d50874c1 100644
> --- a/arch/powerpc/mm/book3s64/slb.c
> +++ b/arch/powerpc/mm/book3s64/slb.c
> @@ -837,8 +837,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)
> return slb_insert_entry(ea, context, flags, ssize, false);
> }
>
> -long do_slb_fault(struct pt_regs *regs, unsigned long ea)
> +long do_slb_fault(struct pt_regs *regs)
> {
> + unsigned long ea = regs->dar;
> unsigned long id = get_region_id(ea);
>
> /* IRQs are not reconciled here, so can't check irqs_disabled */
> @@ -889,13 +890,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea)
> }
> }
>
> -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err)
> +void do_bad_slb_fault(struct pt_regs *regs)
> {
> + int err = regs->result;
> +
> if (err == -EFAULT) {
> if (user_mode(regs))
> - _exception(SIGSEGV, regs, SEGV_BNDERR, ea);
> + _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar);
> else
> - bad_page_fault(regs, ea, SIGSEGV);
> + bad_page_fault(regs, regs->dar, SIGSEGV);
> } else if (err == -EINVAL) {
> unrecoverable_exception(regs);
> } else {
> diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
> index e65a49f246ef..390a296b16a3 100644
> --- a/arch/powerpc/mm/fault.c
> +++ b/arch/powerpc/mm/fault.c
> @@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
> }
> NOKPROBE_SYMBOL(__do_page_fault);
>
> -int do_page_fault(struct pt_regs *regs, unsigned long address,
> - unsigned long error_code)
> +long do_page_fault(struct pt_regs *regs)
> {
> enum ctx_state prev_state = exception_enter();
> - int err;
> + unsigned long address = regs->dar;
> + unsigned long error_code = regs->dsisr;
> + long err;
By doing something more or less like this (need to be tuned for bookE as well):
+ int is_exec = TRAP(regs) == 0x400;
+ unsigned long address = is_exec ? regs->ssr0 : regs->dar;
+ unsigned long error_code = is_exec ? (regs->ssr1 & DSISR_SRR1_MATCH_32S) : regs->dsisr;
>
> err = __do_page_fault(regs, address, error_code);
>
> @@ -580,11 +581,12 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
> NOKPROBE_SYMBOL(do_page_fault);
>
> #ifdef CONFIG_PPC_BOOK3S_64
> -/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */
> -int hash__do_page_fault(struct pt_regs *regs, unsigned long address,
> - unsigned long error_code)
> +/* Same as do_page_fault but no interrupt entry */
> +long hash__do_page_fault(struct pt_regs *regs)
> {
> - int err;
> + unsigned long address = regs->dar;
> + unsigned long error_code = regs->dsisr;
> + long err;
>
> err = __do_page_fault(regs, address, error_code);
> if (unlikely(err)) {
>
There is probably also something we can simplify around get_and_save_dar_dsisr_on_stack() macro in
head_32.h, no need to reload DAR, at least for 8xx. Maybe as a followup patch later.
Christophe
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 02/18] powerpc: remove arguments from fault handler functions
2020-11-06 7:59 ` Christophe Leroy
@ 2020-11-10 8:29 ` Nicholas Piggin
2020-11-10 11:15 ` Christophe Leroy
0 siblings, 1 reply; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-10 8:29 UTC (permalink / raw)
To: Christophe Leroy, linuxppc-dev
Excerpts from Christophe Leroy's message of November 6, 2020 5:59 pm:
>
>
> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>> Make mm fault handlers all just take the pt_regs * argument and load
>> DAR/DSISR from that. Make those that return a value return long.
>>
>> This is done to make the function signatures match other handlers, which
>> will help with a future patch to add wrappers. Explicit arguments could
>> be added for performance but that would require more wrapper macro
>> variants.
>>
>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>> ---
>> arch/powerpc/include/asm/asm-prototypes.h | 4 ++--
>> arch/powerpc/include/asm/bug.h | 4 ++--
>> arch/powerpc/kernel/exceptions-64e.S | 2 --
>> arch/powerpc/kernel/exceptions-64s.S | 14 ++------------
>> arch/powerpc/kernel/head_40x.S | 10 +++++-----
>> arch/powerpc/kernel/head_8xx.S | 6 +++---
>> arch/powerpc/kernel/head_book3s_32.S | 6 ++----
>> arch/powerpc/kernel/head_booke.h | 4 +---
>> arch/powerpc/mm/book3s64/hash_utils.c | 8 +++++---
>> arch/powerpc/mm/book3s64/slb.c | 11 +++++++----
>> arch/powerpc/mm/fault.c | 16 +++++++++-------
>> 11 files changed, 38 insertions(+), 47 deletions(-)
>>
>> diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
>> index d0b832cbbec8..22c9d08fa3a4 100644
>> --- a/arch/powerpc/include/asm/asm-prototypes.h
>> +++ b/arch/powerpc/include/asm/asm-prototypes.h
>> @@ -82,8 +82,8 @@ void kernel_bad_stack(struct pt_regs *regs);
>> void system_reset_exception(struct pt_regs *regs);
>> void machine_check_exception(struct pt_regs *regs);
>> void emulation_assist_interrupt(struct pt_regs *regs);
>> -long do_slb_fault(struct pt_regs *regs, unsigned long ea);
>> -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err);
>> +long do_slb_fault(struct pt_regs *regs);
>> +void do_bad_slb_fault(struct pt_regs *regs);
>>
>> /* signals, syscalls and interrupts */
>> long sys_swapcontext(struct ucontext __user *old_ctx,
>> diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
>> index d714d83bbc7c..2fa0cf6c6011 100644
>> --- a/arch/powerpc/include/asm/bug.h
>> +++ b/arch/powerpc/include/asm/bug.h
>> @@ -111,8 +111,8 @@
>> #ifndef __ASSEMBLY__
>>
>> struct pt_regs;
>> -extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long);
>> -extern int hash__do_page_fault(struct pt_regs *, unsigned long, unsigned long);
>> +extern long do_page_fault(struct pt_regs *);
>> +extern long hash__do_page_fault(struct pt_regs *);
>
> extern is pointless
Thanks. Sorry I'll get it right one day.
>> @@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit)
>> */
>> START_EXCEPTION(0x0400, InstructionAccess)
>> EXCEPTION_PROLOG
>> - mr r4,r12 /* Pass SRR0 as arg2 */
>> - stw r4, _DEAR(r11)
>> - li r5,0 /* Pass zero as arg3 */
>> + li r5,0
>> + stw r5, _ESR(r11) /* Zero ESR */
>> + stw r12, _DEAR(r11) /* SRR0 as DEAR */
>
> I think we should avoid this, see below
>
>> @@ -356,14 +356,14 @@ DataStoreTLBMiss:
>> . = 0x1300
>> InstructionTLBError:
>> EXCEPTION_PROLOG
>> - mr r4,r12
>> andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
>
> Could avoid this, see below
>
>> andis. r10,r9,SRR1_ISI_NOPT@h
>> beq+ .Litlbie
>> - tlbie r4
>> + tlbie r12
>> /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
>> .Litlbie:
>> - stw r4, _DAR(r11)
>> + stw r12, _DAR(r11)
>> + stw r5, _DSISR(r11)
>
> And this
>> @@ -369,9 +369,9 @@ BEGIN_MMU_FTR_SECTION
>> bl hash_page
>> END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
>> #endif /* CONFIG_VMAP_STACK */
>> -1: mr r4,r12
>> andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
>> - stw r4, _DAR(r11)
>> + stw r5, _DSISR(r11)
>> + stw r12, _DAR(r11)
>
> And this including the andis.
>
>> @@ -477,9 +477,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
>> NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \
>> mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
>> stw r5,_ESR(r11); \
>> - mr r4,r12; /* Pass SRR0 as arg2 */ \
>> - stw r4, _DEAR(r11); \
>> - li r5,0; /* Pass zero as arg3 */ \
>> + stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
>
> And this
>
[...]
>> diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
>> index e65a49f246ef..390a296b16a3 100644
>> --- a/arch/powerpc/mm/fault.c
>> +++ b/arch/powerpc/mm/fault.c
>> @@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
>> }
>> NOKPROBE_SYMBOL(__do_page_fault);
>>
>> -int do_page_fault(struct pt_regs *regs, unsigned long address,
>> - unsigned long error_code)
>> +long do_page_fault(struct pt_regs *regs)
>> {
>> enum ctx_state prev_state = exception_enter();
>> - int err;
>> + unsigned long address = regs->dar;
>> + unsigned long error_code = regs->dsisr;
>> + long err;
>
> By doing something more or less like this (need to be tuned for bookE as well):
>
> + int is_exec = TRAP(regs) == 0x400;
> + unsigned long address = is_exec ? regs->ssr0 : regs->dar;
> + unsigned long error_code = is_exec ? (regs->ssr1 & DSISR_SRR1_MATCH_32S) : regs->dsisr;
Ah, I didn't see that you saved these in srr0/1 already. Hmm, not in
pt_regs though. thread_struct (VMAP_STACK only)? exception_regs (booke
only)? Doesn't seem so easy.
In general I don't have a problem with some processor specific things
like this in do page_fault though if it speeds things up. If it gets
a bit more complicated then we can have some accsssor function
get_fault_details(regs, &address, &error_code, &is_exec);
>> @@ -580,11 +581,12 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
>> NOKPROBE_SYMBOL(do_page_fault);
>>
>> #ifdef CONFIG_PPC_BOOK3S_64
>> -/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */
>> -int hash__do_page_fault(struct pt_regs *regs, unsigned long address,
>> - unsigned long error_code)
>> +/* Same as do_page_fault but no interrupt entry */
>> +long hash__do_page_fault(struct pt_regs *regs)
>> {
>> - int err;
>> + unsigned long address = regs->dar;
>> + unsigned long error_code = regs->dsisr;
>> + long err;
>>
>> err = __do_page_fault(regs, address, error_code);
>> if (unlikely(err)) {
>>
>
> There is probably also something we can simplify around get_and_save_dar_dsisr_on_stack() macro in
> head_32.h, no need to reload DAR, at least for 8xx. Maybe as a followup patch later.
Admittedly my 32-bit knowledge or test environments are not great :(
Thanks,
Nick
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 02/18] powerpc: remove arguments from fault handler functions
2020-11-10 8:29 ` Nicholas Piggin
@ 2020-11-10 11:15 ` Christophe Leroy
2020-11-11 4:45 ` Nicholas Piggin
0 siblings, 1 reply; 36+ messages in thread
From: Christophe Leroy @ 2020-11-10 11:15 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev
Le 10/11/2020 à 09:29, Nicholas Piggin a écrit :
> Excerpts from Christophe Leroy's message of November 6, 2020 5:59 pm:
>>
>>
>> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>>> Make mm fault handlers all just take the pt_regs * argument and load
>>> DAR/DSISR from that. Make those that return a value return long.
>>>
>>> This is done to make the function signatures match other handlers, which
>>> will help with a future patch to add wrappers. Explicit arguments could
>>> be added for performance but that would require more wrapper macro
>>> variants.
>>>
>>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>>> ---
[...]
>
>>> diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
>>> index e65a49f246ef..390a296b16a3 100644
>>> --- a/arch/powerpc/mm/fault.c
>>> +++ b/arch/powerpc/mm/fault.c
>>> @@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
>>> }
>>> NOKPROBE_SYMBOL(__do_page_fault);
>>>
>>> -int do_page_fault(struct pt_regs *regs, unsigned long address,
>>> - unsigned long error_code)
>>> +long do_page_fault(struct pt_regs *regs)
>>> {
>>> enum ctx_state prev_state = exception_enter();
>>> - int err;
>>> + unsigned long address = regs->dar;
>>> + unsigned long error_code = regs->dsisr;
>>> + long err;
>>
>> By doing something more or less like this (need to be tuned for bookE as well):
>>
>> + int is_exec = TRAP(regs) == 0x400;
>> + unsigned long address = is_exec ? regs->ssr0 : regs->dar;
>> + unsigned long error_code = is_exec ? (regs->ssr1 & DSISR_SRR1_MATCH_32S) : regs->dsisr;
>
> Ah, I didn't see that you saved these in srr0/1 already. Hmm, not in
> pt_regs though. thread_struct (VMAP_STACK only)? exception_regs (booke
> only)? Doesn't seem so easy.
Oops yes you are right, SRR0/SRR1 are not in pt_regs. And their validity in thread struct is rather
short ... So forget my comment.
Christophe
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 02/18] powerpc: remove arguments from fault handler functions
2020-11-10 11:15 ` Christophe Leroy
@ 2020-11-11 4:45 ` Nicholas Piggin
0 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-11 4:45 UTC (permalink / raw)
To: Christophe Leroy, linuxppc-dev
Excerpts from Christophe Leroy's message of November 10, 2020 9:15 pm:
>
>
> Le 10/11/2020 à 09:29, Nicholas Piggin a écrit :
>> Excerpts from Christophe Leroy's message of November 6, 2020 5:59 pm:
>>>
>>>
>>> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>>>> Make mm fault handlers all just take the pt_regs * argument and load
>>>> DAR/DSISR from that. Make those that return a value return long.
>>>>
>>>> This is done to make the function signatures match other handlers, which
>>>> will help with a future patch to add wrappers. Explicit arguments could
>>>> be added for performance but that would require more wrapper macro
>>>> variants.
>>>>
>>>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>>>> ---
>
> [...]
>
>>
>>>> diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
>>>> index e65a49f246ef..390a296b16a3 100644
>>>> --- a/arch/powerpc/mm/fault.c
>>>> +++ b/arch/powerpc/mm/fault.c
>>>> @@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
>>>> }
>>>> NOKPROBE_SYMBOL(__do_page_fault);
>>>>
>>>> -int do_page_fault(struct pt_regs *regs, unsigned long address,
>>>> - unsigned long error_code)
>>>> +long do_page_fault(struct pt_regs *regs)
>>>> {
>>>> enum ctx_state prev_state = exception_enter();
>>>> - int err;
>>>> + unsigned long address = regs->dar;
>>>> + unsigned long error_code = regs->dsisr;
>>>> + long err;
>>>
>>> By doing something more or less like this (need to be tuned for bookE as well):
>>>
>>> + int is_exec = TRAP(regs) == 0x400;
>>> + unsigned long address = is_exec ? regs->ssr0 : regs->dar;
>>> + unsigned long error_code = is_exec ? (regs->ssr1 & DSISR_SRR1_MATCH_32S) : regs->dsisr;
>>
>> Ah, I didn't see that you saved these in srr0/1 already. Hmm, not in
>> pt_regs though. thread_struct (VMAP_STACK only)? exception_regs (booke
>> only)? Doesn't seem so easy.
>
> Oops yes you are right, SRR0/SRR1 are not in pt_regs. And their validity in thread struct is rather
> short ... So forget my comment.
So, are you happy to go with this for now? I guess things can
later be cleaned up to avoid double saving on cases like VMAP.
Thanks,
Nick
^ permalink raw reply [flat|nested] 36+ messages in thread
* [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
2020-11-05 14:34 ` [PATCH 01/18] powerpc/64s: move the last of the page fault handling logic to C Nicholas Piggin
2020-11-05 14:34 ` [PATCH 02/18] powerpc: remove arguments from fault handler functions Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 20:43 ` kernel test robot
2020-11-06 8:14 ` Christophe Leroy
2020-11-05 14:34 ` [PATCH 04/18] powerpc: interrupt handler wrapper functions Nicholas Piggin
` (14 subsequent siblings)
17 siblings, 2 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
This also moves the 32s DABR match to C.
Similar to the previous patch this makes interrupt handler function
types more regular so they can be wrapped with the next patch.
bad_page_fault and do_break are not performance critical.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/bug.h | 2 +-
arch/powerpc/include/asm/debug.h | 3 +--
arch/powerpc/kernel/entry_32.S | 14 ++++----------
arch/powerpc/kernel/exceptions-64e.S | 3 +--
arch/powerpc/kernel/exceptions-64s.S | 3 +--
arch/powerpc/kernel/head_8xx.S | 5 ++---
arch/powerpc/kernel/process.c | 7 +++----
arch/powerpc/kernel/traps.c | 2 +-
arch/powerpc/mm/book3s64/hash_utils.c | 4 ++--
arch/powerpc/mm/book3s64/slb.c | 2 +-
arch/powerpc/mm/fault.c | 14 +++++++-------
arch/powerpc/platforms/8xx/machine_check.c | 2 +-
12 files changed, 25 insertions(+), 36 deletions(-)
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index 2fa0cf6c6011..4af6c3835eb2 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -113,7 +113,7 @@
struct pt_regs;
extern long do_page_fault(struct pt_regs *);
extern long hash__do_page_fault(struct pt_regs *);
-extern void bad_page_fault(struct pt_regs *, unsigned long, int);
+extern void bad_page_fault(struct pt_regs *, int);
extern void _exception(int, struct pt_regs *, int, unsigned long);
extern void _exception_pkey(struct pt_regs *, unsigned long, int);
extern void die(const char *, struct pt_regs *, long);
diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h
index ec57daf87f40..0550eceab3ca 100644
--- a/arch/powerpc/include/asm/debug.h
+++ b/arch/powerpc/include/asm/debug.h
@@ -52,8 +52,7 @@ extern void do_send_trap(struct pt_regs *regs, unsigned long address,
unsigned long error_code, int brkpt);
#else
-extern void do_break(struct pt_regs *regs, unsigned long address,
- unsigned long error_code);
+void do_break(struct pt_regs *regs);
#endif
#endif /* _ASM_POWERPC_DEBUG_H */
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 8cdc8bcde703..eb97df234a0c 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -657,10 +657,6 @@ ppc_swapcontext:
.globl handle_page_fault
handle_page_fault:
addi r3,r1,STACK_FRAME_OVERHEAD
-#ifdef CONFIG_PPC_BOOK3S_32
- andis. r0,r5,DSISR_DABRMATCH@h
- bne- handle_dabr_fault
-#endif
bl do_page_fault
cmpwi r3,0
beq+ ret_from_except
@@ -668,19 +664,17 @@ handle_page_fault:
lwz r0,_TRAP(r1)
clrrwi r0,r0,1
stw r0,_TRAP(r1)
- mr r5,r3
+ mr r4,r3 /* err arg for bad_page_fault */
addi r3,r1,STACK_FRAME_OVERHEAD
- lwz r4,_DAR(r1)
+#ifdef CONFIG_PPC_BOOK3S_32
+ blt handle_dabr_fault
+#endif
bl bad_page_fault
b ret_from_except_full
#ifdef CONFIG_PPC_BOOK3S_32
/* We have a data breakpoint exception - handle it */
handle_dabr_fault:
- SAVE_NVGPRS(r1)
- lwz r0,_TRAP(r1)
- clrrwi r0,r0,1
- stw r0,_TRAP(r1)
bl do_break
b ret_from_except_full
#endif
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 25fa7d5a643c..dc728bb1c89a 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -1018,9 +1018,8 @@ storage_fault_common:
bne- 1f
b ret_from_except_lite
1: bl save_nvgprs
- mr r5,r3
+ mr r4,r3
addi r3,r1,STACK_FRAME_OVERHEAD
- ld r4,_DAR(r1)
bl bad_page_fault
b ret_from_except
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 1f34cfd1887c..e6558c4d3f81 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -2135,8 +2135,7 @@ EXC_COMMON_BEGIN(h_data_storage_common)
GEN_COMMON h_data_storage
addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
- ld r4,_DAR(r1)
- li r5,SIGSEGV
+ li r4,SIGSEGV
bl bad_page_fault
MMU_FTR_SECTION_ELSE
bl unknown_exception
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 0cd95b633e2b..13eda7154695 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -408,10 +408,9 @@ do_databreakpoint:
addi r3,r1,STACK_FRAME_OVERHEAD
mfspr r4,SPRN_BAR
stw r4,_DAR(r11)
-#ifdef CONFIG_VMAP_STACK
- lwz r5,_DSISR(r11)
-#else
+#ifndef CONFIG_VMAP_STACK
mfspr r5,SPRN_DSISR
+ stw r5,_DSISR(r11)
#endif
EXC_XFER_STD(0x1c00, do_break)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index d421a2c7f822..0bdd3ed653df 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -660,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs)
}
}
-void do_break (struct pt_regs *regs, unsigned long address,
- unsigned long error_code)
+void do_break(struct pt_regs *regs)
{
current->thread.trap_nr = TRAP_HWBKPT;
- if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
+ if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, regs->dsisr,
11, SIGSEGV) == NOTIFY_STOP)
return;
@@ -682,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address,
do_break_handler(regs);
/* Deliver the signal to userspace */
- force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address);
+ force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar);
}
#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 5006dcbe1d9f..902fcbd1a778 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1641,7 +1641,7 @@ void alignment_exception(struct pt_regs *regs)
if (user_mode(regs))
_exception(sig, regs, code, regs->dar);
else
- bad_page_fault(regs, regs->dar, sig);
+ bad_page_fault(regs, sig);
bail:
exception_exit(prev_state);
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index 0f0bd4af4b2d..731518e7d56f 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -1537,7 +1537,7 @@ long do_hash_fault(struct pt_regs *regs)
* the access, or panic if there isn't a handler.
*/
if (unlikely(in_nmi())) {
- bad_page_fault(regs, ea, SIGSEGV);
+ bad_page_fault(regs, SIGSEGV);
return 0;
}
@@ -1576,7 +1576,7 @@ long do_hash_fault(struct pt_regs *regs)
else
_exception(SIGBUS, regs, BUS_ADRERR, ea);
} else {
- bad_page_fault(regs, ea, SIGBUS);
+ bad_page_fault(regs, SIGBUS);
}
err = 0;
diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c
index cc34d50874c1..ae89ad516247 100644
--- a/arch/powerpc/mm/book3s64/slb.c
+++ b/arch/powerpc/mm/book3s64/slb.c
@@ -898,7 +898,7 @@ void do_bad_slb_fault(struct pt_regs *regs)
if (user_mode(regs))
_exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar);
else
- bad_page_fault(regs, regs->dar, SIGSEGV);
+ bad_page_fault(regs, SIGSEGV);
} else if (err == -EINVAL) {
unrecoverable_exception(regs);
} else {
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 390a296b16a3..49fbe564ea2b 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -375,7 +375,7 @@ static void sanity_check_fault(bool is_write, bool is_user,
#elif defined(CONFIG_PPC_BOOK3E_64)
#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S)
#else
-#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S)
+#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)
#endif
#endif
@@ -408,7 +408,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
return 0;
if (unlikely(page_fault_is_bad(error_code))) {
- if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && (error_code & DSISR_DABRMATCH))
+ if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (error_code & DSISR_DABRMATCH))
return -1;
if (is_user) {
@@ -562,14 +562,14 @@ long do_page_fault(struct pt_regs *regs)
/* 32 and 64e handle errors in their asm code */
if (unlikely(err)) {
if (err > 0) {
- bad_page_fault(regs, address, err);
+ bad_page_fault(regs, err);
err = 0;
} else {
/*
* do_break() may change NV GPRS while handling the
* breakpoint. Return -ve to caller to do that.
*/
- do_break(regs, address, error_code);
+ do_break(regs);
}
}
#endif
@@ -591,14 +591,14 @@ long hash__do_page_fault(struct pt_regs *regs)
err = __do_page_fault(regs, address, error_code);
if (unlikely(err)) {
if (err > 0) {
- bad_page_fault(regs, address, err);
+ bad_page_fault(regs, err);
err = 0;
} else {
/*
* do_break() may change NV GPRS while handling the
* breakpoint. Return -ve to caller to do that.
*/
- do_break(regs, address, error_code);
+ do_break(regs);
}
}
@@ -612,7 +612,7 @@ NOKPROBE_SYMBOL(hash__do_page_fault);
* It is called from the DSI and ISI handlers in head.S and from some
* of the procedures in traps.c.
*/
-void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
+void bad_page_fault(struct pt_regs *regs, int sig)
{
const struct exception_table_entry *entry;
int is_write = page_fault_is_write(regs->dsisr);
diff --git a/arch/powerpc/platforms/8xx/machine_check.c b/arch/powerpc/platforms/8xx/machine_check.c
index 88dedf38eccd..656365975895 100644
--- a/arch/powerpc/platforms/8xx/machine_check.c
+++ b/arch/powerpc/platforms/8xx/machine_check.c
@@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs)
* to deal with that than having a wart in the mcheck handler.
* -- BenH
*/
- bad_page_fault(regs, regs->dar, SIGBUS);
+ bad_page_fault(regs, SIGBUS);
return 1;
#else
return 0;
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-05 14:34 ` [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs Nicholas Piggin
@ 2020-11-05 20:43 ` kernel test robot
2020-11-06 8:14 ` Christophe Leroy
1 sibling, 0 replies; 36+ messages in thread
From: kernel test robot @ 2020-11-05 20:43 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev; +Cc: kbuild-all, Nicholas Piggin
[-- Attachment #1: Type: text/plain, Size: 3374 bytes --]
Hi Nicholas,
I love your patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on v5.10-rc2 next-20201105]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-allnoconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/4232a616cd2a8f7ef6b3f19cd656690dc5ec4c9e
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
git checkout 4232a616cd2a8f7ef6b3f19cd656690dc5ec4c9e
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
In file included from include/asm-generic/bug.h:5,
from arch/powerpc/include/asm/bug.h:109,
from include/linux/bug.h:5,
from include/linux/signal.h:5,
from arch/powerpc/mm/fault.c:14:
arch/powerpc/mm/fault.c: In function '__do_page_fault':
>> arch/powerpc/mm/fault.c:378:43: error: suggest parentheses around arithmetic in operand of '|' [-Werror=parentheses]
378 | #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)
include/linux/compiler.h:78:42: note: in definition of macro 'unlikely'
78 | # define unlikely(x) __builtin_expect(!!(x), 0)
| ^
arch/powerpc/mm/fault.c:410:15: note: in expansion of macro 'page_fault_is_bad'
410 | if (unlikely(page_fault_is_bad(error_code))) {
| ^~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
vim +378 arch/powerpc/mm/fault.c
361
362 /*
363 * Define the correct "is_write" bit in error_code based
364 * on the processor family
365 */
366 #if (defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
367 #define page_fault_is_write(__err) ((__err) & ESR_DST)
368 #define page_fault_is_bad(__err) (0)
369 #else
370 #define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE)
371 #if defined(CONFIG_PPC_8xx)
372 #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G)
373 #elif defined(CONFIG_PPC_BOOK3S_64)
374 #define page_fault_is_bad(__err) ((__err) & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH))
375 #elif defined(CONFIG_PPC_BOOK3E_64)
376 #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S)
377 #else
> 378 #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)
379 #endif
380 #endif
381
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 6452 bytes --]
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
@ 2020-11-05 20:43 ` kernel test robot
0 siblings, 0 replies; 36+ messages in thread
From: kernel test robot @ 2020-11-05 20:43 UTC (permalink / raw)
To: kbuild-all
[-- Attachment #1: Type: text/plain, Size: 3448 bytes --]
Hi Nicholas,
I love your patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on v5.10-rc2 next-20201105]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-allnoconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/4232a616cd2a8f7ef6b3f19cd656690dc5ec4c9e
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
git checkout 4232a616cd2a8f7ef6b3f19cd656690dc5ec4c9e
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
In file included from include/asm-generic/bug.h:5,
from arch/powerpc/include/asm/bug.h:109,
from include/linux/bug.h:5,
from include/linux/signal.h:5,
from arch/powerpc/mm/fault.c:14:
arch/powerpc/mm/fault.c: In function '__do_page_fault':
>> arch/powerpc/mm/fault.c:378:43: error: suggest parentheses around arithmetic in operand of '|' [-Werror=parentheses]
378 | #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)
include/linux/compiler.h:78:42: note: in definition of macro 'unlikely'
78 | # define unlikely(x) __builtin_expect(!!(x), 0)
| ^
arch/powerpc/mm/fault.c:410:15: note: in expansion of macro 'page_fault_is_bad'
410 | if (unlikely(page_fault_is_bad(error_code))) {
| ^~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
vim +378 arch/powerpc/mm/fault.c
361
362 /*
363 * Define the correct "is_write" bit in error_code based
364 * on the processor family
365 */
366 #if (defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
367 #define page_fault_is_write(__err) ((__err) & ESR_DST)
368 #define page_fault_is_bad(__err) (0)
369 #else
370 #define page_fault_is_write(__err) ((__err) & DSISR_ISSTORE)
371 #if defined(CONFIG_PPC_8xx)
372 #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G)
373 #elif defined(CONFIG_PPC_BOOK3S_64)
374 #define page_fault_is_bad(__err) ((__err) & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH))
375 #elif defined(CONFIG_PPC_BOOK3E_64)
376 #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S)
377 #else
> 378 #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)
379 #endif
380 #endif
381
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 6452 bytes --]
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-05 14:34 ` [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs Nicholas Piggin
2020-11-05 20:43 ` kernel test robot
@ 2020-11-06 8:14 ` Christophe Leroy
2020-11-10 8:34 ` Nicholas Piggin
1 sibling, 1 reply; 36+ messages in thread
From: Christophe Leroy @ 2020-11-06 8:14 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev
Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
> This also moves the 32s DABR match to C.
Is there a real benefit doing this ?
>
> Similar to the previous patch this makes interrupt handler function
> types more regular so they can be wrapped with the next patch.
>
> bad_page_fault and do_break are not performance critical.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> ---
> arch/powerpc/include/asm/bug.h | 2 +-
> arch/powerpc/include/asm/debug.h | 3 +--
> arch/powerpc/kernel/entry_32.S | 14 ++++----------
> arch/powerpc/kernel/exceptions-64e.S | 3 +--
> arch/powerpc/kernel/exceptions-64s.S | 3 +--
> arch/powerpc/kernel/head_8xx.S | 5 ++---
> arch/powerpc/kernel/process.c | 7 +++----
> arch/powerpc/kernel/traps.c | 2 +-
> arch/powerpc/mm/book3s64/hash_utils.c | 4 ++--
> arch/powerpc/mm/book3s64/slb.c | 2 +-
> arch/powerpc/mm/fault.c | 14 +++++++-------
> arch/powerpc/platforms/8xx/machine_check.c | 2 +-
> 12 files changed, 25 insertions(+), 36 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
> index 2fa0cf6c6011..4af6c3835eb2 100644
> --- a/arch/powerpc/include/asm/bug.h
> +++ b/arch/powerpc/include/asm/bug.h
> @@ -113,7 +113,7 @@
> struct pt_regs;
> extern long do_page_fault(struct pt_regs *);
> extern long hash__do_page_fault(struct pt_regs *);
> -extern void bad_page_fault(struct pt_regs *, unsigned long, int);
> +extern void bad_page_fault(struct pt_regs *, int);
pointless extern
Christophe
> extern void _exception(int, struct pt_regs *, int, unsigned long);
> extern void _exception_pkey(struct pt_regs *, unsigned long, int);
> extern void die(const char *, struct pt_regs *, long);
> diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h
> index ec57daf87f40..0550eceab3ca 100644
> --- a/arch/powerpc/include/asm/debug.h
> +++ b/arch/powerpc/include/asm/debug.h
> @@ -52,8 +52,7 @@ extern void do_send_trap(struct pt_regs *regs, unsigned long address,
> unsigned long error_code, int brkpt);
> #else
>
> -extern void do_break(struct pt_regs *regs, unsigned long address,
> - unsigned long error_code);
> +void do_break(struct pt_regs *regs);
> #endif
>
> #endif /* _ASM_POWERPC_DEBUG_H */
> diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
> index 8cdc8bcde703..eb97df234a0c 100644
> --- a/arch/powerpc/kernel/entry_32.S
> +++ b/arch/powerpc/kernel/entry_32.S
> @@ -657,10 +657,6 @@ ppc_swapcontext:
> .globl handle_page_fault
> handle_page_fault:
> addi r3,r1,STACK_FRAME_OVERHEAD
> -#ifdef CONFIG_PPC_BOOK3S_32
> - andis. r0,r5,DSISR_DABRMATCH@h
> - bne- handle_dabr_fault
> -#endif
> bl do_page_fault
> cmpwi r3,0
> beq+ ret_from_except
> @@ -668,19 +664,17 @@ handle_page_fault:
> lwz r0,_TRAP(r1)
> clrrwi r0,r0,1
> stw r0,_TRAP(r1)
> - mr r5,r3
> + mr r4,r3 /* err arg for bad_page_fault */
> addi r3,r1,STACK_FRAME_OVERHEAD
> - lwz r4,_DAR(r1)
> +#ifdef CONFIG_PPC_BOOK3S_32
> + blt handle_dabr_fault
> +#endif
> bl bad_page_fault
> b ret_from_except_full
>
> #ifdef CONFIG_PPC_BOOK3S_32
> /* We have a data breakpoint exception - handle it */
> handle_dabr_fault:
> - SAVE_NVGPRS(r1)
> - lwz r0,_TRAP(r1)
> - clrrwi r0,r0,1
> - stw r0,_TRAP(r1)
> bl do_break
> b ret_from_except_full
> #endif
> diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
> index 25fa7d5a643c..dc728bb1c89a 100644
> --- a/arch/powerpc/kernel/exceptions-64e.S
> +++ b/arch/powerpc/kernel/exceptions-64e.S
> @@ -1018,9 +1018,8 @@ storage_fault_common:
> bne- 1f
> b ret_from_except_lite
> 1: bl save_nvgprs
> - mr r5,r3
> + mr r4,r3
> addi r3,r1,STACK_FRAME_OVERHEAD
> - ld r4,_DAR(r1)
> bl bad_page_fault
> b ret_from_except
>
> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> index 1f34cfd1887c..e6558c4d3f81 100644
> --- a/arch/powerpc/kernel/exceptions-64s.S
> +++ b/arch/powerpc/kernel/exceptions-64s.S
> @@ -2135,8 +2135,7 @@ EXC_COMMON_BEGIN(h_data_storage_common)
> GEN_COMMON h_data_storage
> addi r3,r1,STACK_FRAME_OVERHEAD
> BEGIN_MMU_FTR_SECTION
> - ld r4,_DAR(r1)
> - li r5,SIGSEGV
> + li r4,SIGSEGV
> bl bad_page_fault
> MMU_FTR_SECTION_ELSE
> bl unknown_exception
> diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
> index 0cd95b633e2b..13eda7154695 100644
> --- a/arch/powerpc/kernel/head_8xx.S
> +++ b/arch/powerpc/kernel/head_8xx.S
> @@ -408,10 +408,9 @@ do_databreakpoint:
> addi r3,r1,STACK_FRAME_OVERHEAD
> mfspr r4,SPRN_BAR
> stw r4,_DAR(r11)
> -#ifdef CONFIG_VMAP_STACK
> - lwz r5,_DSISR(r11)
> -#else
> +#ifndef CONFIG_VMAP_STACK
> mfspr r5,SPRN_DSISR
> + stw r5,_DSISR(r11)
> #endif
> EXC_XFER_STD(0x1c00, do_break)
>
> diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
> index d421a2c7f822..0bdd3ed653df 100644
> --- a/arch/powerpc/kernel/process.c
> +++ b/arch/powerpc/kernel/process.c
> @@ -660,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs)
> }
> }
>
> -void do_break (struct pt_regs *regs, unsigned long address,
> - unsigned long error_code)
> +void do_break(struct pt_regs *regs)
> {
> current->thread.trap_nr = TRAP_HWBKPT;
> - if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
> + if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, regs->dsisr,
> 11, SIGSEGV) == NOTIFY_STOP)
> return;
>
> @@ -682,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address,
> do_break_handler(regs);
>
> /* Deliver the signal to userspace */
> - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address);
> + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar);
> }
> #endif /* CONFIG_PPC_ADV_DEBUG_REGS */
>
> diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
> index 5006dcbe1d9f..902fcbd1a778 100644
> --- a/arch/powerpc/kernel/traps.c
> +++ b/arch/powerpc/kernel/traps.c
> @@ -1641,7 +1641,7 @@ void alignment_exception(struct pt_regs *regs)
> if (user_mode(regs))
> _exception(sig, regs, code, regs->dar);
> else
> - bad_page_fault(regs, regs->dar, sig);
> + bad_page_fault(regs, sig);
>
> bail:
> exception_exit(prev_state);
> diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
> index 0f0bd4af4b2d..731518e7d56f 100644
> --- a/arch/powerpc/mm/book3s64/hash_utils.c
> +++ b/arch/powerpc/mm/book3s64/hash_utils.c
> @@ -1537,7 +1537,7 @@ long do_hash_fault(struct pt_regs *regs)
> * the access, or panic if there isn't a handler.
> */
> if (unlikely(in_nmi())) {
> - bad_page_fault(regs, ea, SIGSEGV);
> + bad_page_fault(regs, SIGSEGV);
> return 0;
> }
>
> @@ -1576,7 +1576,7 @@ long do_hash_fault(struct pt_regs *regs)
> else
> _exception(SIGBUS, regs, BUS_ADRERR, ea);
> } else {
> - bad_page_fault(regs, ea, SIGBUS);
> + bad_page_fault(regs, SIGBUS);
> }
> err = 0;
>
> diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c
> index cc34d50874c1..ae89ad516247 100644
> --- a/arch/powerpc/mm/book3s64/slb.c
> +++ b/arch/powerpc/mm/book3s64/slb.c
> @@ -898,7 +898,7 @@ void do_bad_slb_fault(struct pt_regs *regs)
> if (user_mode(regs))
> _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar);
> else
> - bad_page_fault(regs, regs->dar, SIGSEGV);
> + bad_page_fault(regs, SIGSEGV);
> } else if (err == -EINVAL) {
> unrecoverable_exception(regs);
> } else {
> diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
> index 390a296b16a3..49fbe564ea2b 100644
> --- a/arch/powerpc/mm/fault.c
> +++ b/arch/powerpc/mm/fault.c
> @@ -375,7 +375,7 @@ static void sanity_check_fault(bool is_write, bool is_user,
> #elif defined(CONFIG_PPC_BOOK3E_64)
> #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S)
> #else
> -#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S)
> +#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)
> #endif
> #endif
>
> @@ -408,7 +408,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
> return 0;
>
> if (unlikely(page_fault_is_bad(error_code))) {
> - if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && (error_code & DSISR_DABRMATCH))
> + if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (error_code & DSISR_DABRMATCH))
> return -1;
>
> if (is_user) {
> @@ -562,14 +562,14 @@ long do_page_fault(struct pt_regs *regs)
> /* 32 and 64e handle errors in their asm code */
> if (unlikely(err)) {
> if (err > 0) {
> - bad_page_fault(regs, address, err);
> + bad_page_fault(regs, err);
> err = 0;
> } else {
> /*
> * do_break() may change NV GPRS while handling the
> * breakpoint. Return -ve to caller to do that.
> */
> - do_break(regs, address, error_code);
> + do_break(regs);
> }
> }
> #endif
> @@ -591,14 +591,14 @@ long hash__do_page_fault(struct pt_regs *regs)
> err = __do_page_fault(regs, address, error_code);
> if (unlikely(err)) {
> if (err > 0) {
> - bad_page_fault(regs, address, err);
> + bad_page_fault(regs, err);
> err = 0;
> } else {
> /*
> * do_break() may change NV GPRS while handling the
> * breakpoint. Return -ve to caller to do that.
> */
> - do_break(regs, address, error_code);
> + do_break(regs);
> }
> }
>
> @@ -612,7 +612,7 @@ NOKPROBE_SYMBOL(hash__do_page_fault);
> * It is called from the DSI and ISI handlers in head.S and from some
> * of the procedures in traps.c.
> */
> -void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
> +void bad_page_fault(struct pt_regs *regs, int sig)
> {
> const struct exception_table_entry *entry;
> int is_write = page_fault_is_write(regs->dsisr);
> diff --git a/arch/powerpc/platforms/8xx/machine_check.c b/arch/powerpc/platforms/8xx/machine_check.c
> index 88dedf38eccd..656365975895 100644
> --- a/arch/powerpc/platforms/8xx/machine_check.c
> +++ b/arch/powerpc/platforms/8xx/machine_check.c
> @@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs)
> * to deal with that than having a wart in the mcheck handler.
> * -- BenH
> */
> - bad_page_fault(regs, regs->dar, SIGBUS);
> + bad_page_fault(regs, SIGBUS);
> return 1;
> #else
> return 0;
>
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-06 8:14 ` Christophe Leroy
@ 2020-11-10 8:34 ` Nicholas Piggin
2020-11-10 11:19 ` Christophe Leroy
0 siblings, 1 reply; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-10 8:34 UTC (permalink / raw)
To: Christophe Leroy, linuxppc-dev
Excerpts from Christophe Leroy's message of November 6, 2020 6:14 pm:
>
>
> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>> This also moves the 32s DABR match to C.
>
> Is there a real benefit doing this ?
Oh I missed doing it, but yes I think bad_page_fault and do_break should
probably be implemented with the DEFINE_INTERRUT_HANDLER wrappers.
Thanks,
Nick
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-10 8:34 ` Nicholas Piggin
@ 2020-11-10 11:19 ` Christophe Leroy
2020-11-11 4:46 ` Nicholas Piggin
0 siblings, 1 reply; 36+ messages in thread
From: Christophe Leroy @ 2020-11-10 11:19 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev
Le 10/11/2020 à 09:34, Nicholas Piggin a écrit :
> Excerpts from Christophe Leroy's message of November 6, 2020 6:14 pm:
>>
>>
>> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>>> This also moves the 32s DABR match to C.
>>
>> Is there a real benefit doing this ?
>
> Oh I missed doing it, but yes I think bad_page_fault and do_break should
> probably be implemented with the DEFINE_INTERRUT_HANDLER wrappers.
>
Yes, anyway, do we need to do that change ? Can't the dispatch between do_break() and page fault
handling remain in handle_page_fault() ? What's the benefit of going into do_page_fault() and coming
back ?
Christophe
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-10 11:19 ` Christophe Leroy
@ 2020-11-11 4:46 ` Nicholas Piggin
2020-11-11 6:39 ` Nicholas Piggin
0 siblings, 1 reply; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-11 4:46 UTC (permalink / raw)
To: Christophe Leroy, linuxppc-dev
Excerpts from Christophe Leroy's message of November 10, 2020 9:19 pm:
>
>
> Le 10/11/2020 à 09:34, Nicholas Piggin a écrit :
>> Excerpts from Christophe Leroy's message of November 6, 2020 6:14 pm:
>>>
>>>
>>> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>>>> This also moves the 32s DABR match to C.
>>>
>>> Is there a real benefit doing this ?
>>
>> Oh I missed doing it, but yes I think bad_page_fault and do_break should
>> probably be implemented with the DEFINE_INTERRUT_HANDLER wrappers.
>>
>
> Yes, anyway, do we need to do that change ? Can't the dispatch between do_break() and page fault
> handling remain in handle_page_fault() ? What's the benefit of going into do_page_fault() and coming
> back ?
You might be right, I'll take another look at it.
Thanks,
Nick
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs
2020-11-11 4:46 ` Nicholas Piggin
@ 2020-11-11 6:39 ` Nicholas Piggin
0 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-11 6:39 UTC (permalink / raw)
To: Christophe Leroy, linuxppc-dev
Excerpts from Nicholas Piggin's message of November 11, 2020 2:46 pm:
> Excerpts from Christophe Leroy's message of November 10, 2020 9:19 pm:
>>
>>
>> Le 10/11/2020 à 09:34, Nicholas Piggin a écrit :
>>> Excerpts from Christophe Leroy's message of November 6, 2020 6:14 pm:
>>>>
>>>>
>>>> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>>>>> This also moves the 32s DABR match to C.
>>>>
>>>> Is there a real benefit doing this ?
>>>
>>> Oh I missed doing it, but yes I think bad_page_fault and do_break should
>>> probably be implemented with the DEFINE_INTERRUT_HANDLER wrappers.
>>>
>>
>> Yes, anyway, do we need to do that change ? Can't the dispatch between do_break() and page fault
>> handling remain in handle_page_fault() ? What's the benefit of going into do_page_fault() and coming
>> back ?
>
> You might be right, I'll take another look at it.
For 32-bit, we need to come back to save NV GPRs. Certainly the 64s code
stays in do_page_fault because it always saves them.
Now I don't think that's the nicest thing to go in and out of the
interrupt wrappers twice in these cases, but for a first pass I think
it's okay. Either we could add another type of error-case wrapper that
does some adjustment if it becomes necessary, or we find a nice way to
save NVGPRs from C code.
If we could somehow parse unwind data to find where the NVGPRs are saved
by the compiler and generate a little code stub to load them out, would
be the ultimate :) Maybe one day...
Thanks,
Nick
^ permalink raw reply [flat|nested] 36+ messages in thread
* [PATCH 04/18] powerpc: interrupt handler wrapper functions
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (2 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 03/18] powerpc: bad_page_fault, do_break get registers from regs Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 05/18] powerpc: add interrupt wrapper entry / exit stub functions Nicholas Piggin
` (13 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Add wrapper functions (derived from x86 macros) for interrupt handler
functions. This allows interrupt entry code to be written in C.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/asm-prototypes.h | 29 ----
arch/powerpc/include/asm/bug.h | 1 -
arch/powerpc/include/asm/hw_irq.h | 9 -
arch/powerpc/include/asm/interrupt.h | 202 ++++++++++++++++++++++
arch/powerpc/include/asm/time.h | 2 +
arch/powerpc/kernel/dbell.c | 3 +-
arch/powerpc/kernel/exceptions-64s.S | 7 +-
arch/powerpc/kernel/irq.c | 3 +-
arch/powerpc/kernel/mce.c | 5 +-
arch/powerpc/kernel/syscall_64.c | 1 +
arch/powerpc/kernel/tau_6xx.c | 2 +-
arch/powerpc/kernel/time.c | 3 +-
arch/powerpc/kernel/traps.c | 78 ++++++---
arch/powerpc/kernel/watchdog.c | 7 +-
arch/powerpc/kvm/book3s_hv.c | 1 +
arch/powerpc/kvm/book3s_hv_builtin.c | 1 +
arch/powerpc/mm/book3s64/hash_utils.c | 3 +-
arch/powerpc/mm/book3s64/slb.c | 5 +-
arch/powerpc/mm/fault.c | 10 +-
arch/powerpc/platforms/powernv/idle.c | 1 +
20 files changed, 293 insertions(+), 80 deletions(-)
create mode 100644 arch/powerpc/include/asm/interrupt.h
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 22c9d08fa3a4..939f3c94c8f3 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -56,35 +56,6 @@ int exit_vmx_usercopy(void);
int enter_vmx_ops(void);
void *exit_vmx_ops(void *dest);
-/* Traps */
-long machine_check_early(struct pt_regs *regs);
-long hmi_exception_realmode(struct pt_regs *regs);
-void SMIException(struct pt_regs *regs);
-void handle_hmi_exception(struct pt_regs *regs);
-void instruction_breakpoint_exception(struct pt_regs *regs);
-void RunModeException(struct pt_regs *regs);
-void single_step_exception(struct pt_regs *regs);
-void program_check_exception(struct pt_regs *regs);
-void alignment_exception(struct pt_regs *regs);
-void StackOverflow(struct pt_regs *regs);
-void stack_overflow_exception(struct pt_regs *regs);
-void kernel_fp_unavailable_exception(struct pt_regs *regs);
-void altivec_unavailable_exception(struct pt_regs *regs);
-void vsx_unavailable_exception(struct pt_regs *regs);
-void fp_unavailable_tm(struct pt_regs *regs);
-void altivec_unavailable_tm(struct pt_regs *regs);
-void vsx_unavailable_tm(struct pt_regs *regs);
-void facility_unavailable_exception(struct pt_regs *regs);
-void TAUException(struct pt_regs *regs);
-void altivec_assist_exception(struct pt_regs *regs);
-void unrecoverable_exception(struct pt_regs *regs);
-void kernel_bad_stack(struct pt_regs *regs);
-void system_reset_exception(struct pt_regs *regs);
-void machine_check_exception(struct pt_regs *regs);
-void emulation_assist_interrupt(struct pt_regs *regs);
-long do_slb_fault(struct pt_regs *regs);
-void do_bad_slb_fault(struct pt_regs *regs);
-
/* signals, syscalls and interrupts */
long sys_swapcontext(struct ucontext __user *old_ctx,
struct ucontext __user *new_ctx,
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index 4af6c3835eb2..5c74c6ee4cf8 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -111,7 +111,6 @@
#ifndef __ASSEMBLY__
struct pt_regs;
-extern long do_page_fault(struct pt_regs *);
extern long hash__do_page_fault(struct pt_regs *);
extern void bad_page_fault(struct pt_regs *, int);
extern void _exception(int, struct pt_regs *, int, unsigned long);
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 0363734ff56e..614957f74cee 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -50,15 +50,6 @@
#ifndef __ASSEMBLY__
-extern void replay_system_reset(void);
-extern void replay_soft_interrupts(void);
-
-extern void timer_interrupt(struct pt_regs *);
-extern void timer_broadcast_interrupt(void);
-extern void performance_monitor_exception(struct pt_regs *regs);
-extern void WatchdogException(struct pt_regs *regs);
-extern void unknown_exception(struct pt_regs *regs);
-
#ifdef CONFIG_PPC64
#include <asm/paca.h>
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
new file mode 100644
index 000000000000..3292f5094085
--- /dev/null
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -0,0 +1,202 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _ASM_POWERPC_INTERRUPT_H
+#define _ASM_POWERPC_INTERRUPT_H
+
+#include <linux/context_tracking.h>
+#include <asm/ftrace.h>
+
+/**
+ * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function
+ * @func: Function name of the entry point
+ * @returns: Returns a value back to asm caller
+ */
+#define DECLARE_INTERRUPT_HANDLER_RAW(func) \
+ __visible long func(struct pt_regs *regs)
+
+/**
+ * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function
+ * @func: Function name of the entry point
+ * @returns: Returns a value back to asm caller
+ *
+ * @func is called from ASM entry code.
+ *
+ * This is a plain function which does no tracing, reconciling, etc.
+ * The macro is written so it acts as function definition. Append the
+ * body with a pair of curly brackets.
+ */
+#define DEFINE_INTERRUPT_HANDLER_RAW(func) \
+static __always_inline long ___##func(struct pt_regs *regs); \
+ \
+__visible noinstr long func(struct pt_regs *regs) \
+{ \
+ long ret; \
+ \
+ ret = ___##func (regs); \
+ \
+ return ret; \
+} \
+ \
+static __always_inline long ___##func(struct pt_regs *regs)
+
+/**
+ * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function
+ * @func: Function name of the entry point
+ */
+#define DECLARE_INTERRUPT_HANDLER(func) \
+ __visible void func(struct pt_regs *regs)
+
+/**
+ * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function
+ * @func: Function name of the entry point
+ *
+ * @func is called from ASM entry code.
+ *
+ * The macro is written so it acts as function definition. Append the
+ * body with a pair of curly brackets.
+ */
+#define DEFINE_INTERRUPT_HANDLER(func) \
+static __always_inline void ___##func(struct pt_regs *regs); \
+ \
+__visible noinstr void func(struct pt_regs *regs) \
+{ \
+ ___##func (regs); \
+} \
+ \
+static __always_inline void ___##func(struct pt_regs *regs)
+
+/**
+ * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function
+ * @func: Function name of the entry point
+ * @returns: Returns a value back to asm caller
+ */
+#define DECLARE_INTERRUPT_HANDLER_RET(func) \
+ __visible long func(struct pt_regs *regs)
+
+/**
+ * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function
+ * @func: Function name of the entry point
+ * @returns: Returns a value back to asm caller
+ *
+ * @func is called from ASM entry code.
+ *
+ * The macro is written so it acts as function definition. Append the
+ * body with a pair of curly brackets.
+ */
+#define DEFINE_INTERRUPT_HANDLER_RET(func) \
+static __always_inline long ___##func(struct pt_regs *regs); \
+ \
+__visible noinstr long func(struct pt_regs *regs) \
+{ \
+ long ret; \
+ \
+ ret = ___##func (regs); \
+ \
+ return ret; \
+} \
+ \
+static __always_inline long ___##func(struct pt_regs *regs)
+
+/**
+ * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function
+ * @func: Function name of the entry point
+ */
+#define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \
+ __visible void func(struct pt_regs *regs)
+
+/**
+ * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function
+ * @func: Function name of the entry point
+ *
+ * @func is called from ASM entry code.
+ *
+ * The macro is written so it acts as function definition. Append the
+ * body with a pair of curly brackets.
+ */
+#define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \
+static __always_inline void ___##func(struct pt_regs *regs); \
+ \
+__visible noinstr void func(struct pt_regs *regs) \
+{ \
+ ___##func (regs); \
+} \
+ \
+static __always_inline void ___##func(struct pt_regs *regs)
+
+/**
+ * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function
+ * @func: Function name of the entry point
+ * @returns: Returns a value back to asm caller
+ */
+#define DECLARE_INTERRUPT_HANDLER_NMI(func) \
+ __visible long func(struct pt_regs *regs)
+
+/**
+ * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function
+ * @func: Function name of the entry point
+ * @returns: Returns a value back to asm caller
+ *
+ * @func is called from ASM entry code.
+ *
+ * The macro is written so it acts as function definition. Append the
+ * body with a pair of curly brackets.
+ */
+#define DEFINE_INTERRUPT_HANDLER_NMI(func) \
+static __always_inline long ___##func(struct pt_regs *regs); \
+ \
+__visible noinstr long func(struct pt_regs *regs) \
+{ \
+ long ret; \
+ \
+ ret = ___##func (regs); \
+ \
+ return ret; \
+} \
+ \
+static __always_inline long ___##func(struct pt_regs *regs)
+
+
+/* Interrupt handlers */
+DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early);
+DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode);
+DECLARE_INTERRUPT_HANDLER(SMIException);
+DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);
+DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);
+DECLARE_INTERRUPT_HANDLER(RunModeException);
+DECLARE_INTERRUPT_HANDLER(single_step_exception);
+DECLARE_INTERRUPT_HANDLER(program_check_exception);
+DECLARE_INTERRUPT_HANDLER(alignment_exception);
+DECLARE_INTERRUPT_HANDLER(StackOverflow);
+DECLARE_INTERRUPT_HANDLER(stack_overflow_exception);
+DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception);
+DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception);
+DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception);
+DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm);
+DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm);
+DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm);
+DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception);
+DECLARE_INTERRUPT_HANDLER(TAUException);
+DECLARE_INTERRUPT_HANDLER(altivec_assist_exception);
+DECLARE_INTERRUPT_HANDLER(unrecoverable_exception);
+DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);
+DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception);
+#ifdef CONFIG_PPC_BOOK3S_64
+DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception);
+#else
+DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception);
+#endif
+DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt);
+DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault);
+DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault);
+DECLARE_INTERRUPT_HANDLER_RET(do_page_fault);
+DECLARE_INTERRUPT_HANDLER(do_bad_page_fault);
+
+DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt);
+DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception);
+DECLARE_INTERRUPT_HANDLER(WatchdogException);
+DECLARE_INTERRUPT_HANDLER(unknown_exception);
+DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
+
+void replay_system_reset(void);
+void replay_soft_interrupts(void);
+
+#endif /* _ASM_POWERPC_INTERRUPT_H */
diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h
index 2f566c1a754c..335d6fd589a7 100644
--- a/arch/powerpc/include/asm/time.h
+++ b/arch/powerpc/include/asm/time.h
@@ -131,6 +131,8 @@ DECLARE_PER_CPU(u64, decrementers_next_tb);
/* Convert timebase ticks to nanoseconds */
unsigned long long tb_to_ns(unsigned long long tb_ticks);
+void timer_broadcast_interrupt(void);
+
/* SPLPAR */
void accumulate_stolen_time(void);
diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c
index 52680cf07c9d..2c59dee7ec90 100644
--- a/arch/powerpc/kernel/dbell.c
+++ b/arch/powerpc/kernel/dbell.c
@@ -12,13 +12,14 @@
#include <linux/hardirq.h>
#include <asm/dbell.h>
+#include <asm/interrupt.h>
#include <asm/irq_regs.h>
#include <asm/kvm_ppc.h>
#include <asm/trace.h>
#ifdef CONFIG_SMP
-void doorbell_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception)
{
struct pt_regs *old_regs = set_irq_regs(regs);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index e6558c4d3f81..f6989321136d 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1922,7 +1922,7 @@ EXC_COMMON_BEGIN(doorbell_super_common)
#ifdef CONFIG_PPC_DOORBELL
bl doorbell_exception
#else
- bl unknown_exception
+ bl unknown_async_exception
#endif
b interrupt_return
@@ -2135,8 +2135,7 @@ EXC_COMMON_BEGIN(h_data_storage_common)
GEN_COMMON h_data_storage
addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION
- li r4,SIGSEGV
- bl bad_page_fault
+ bl do_bad_page_fault
MMU_FTR_SECTION_ELSE
bl unknown_exception
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
@@ -2309,7 +2308,7 @@ EXC_COMMON_BEGIN(h_doorbell_common)
#ifdef CONFIG_PPC_DOORBELL
bl doorbell_exception
#else
- bl unknown_exception
+ bl unknown_async_exception
#endif
b interrupt_return
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 7d0f7682d01d..c8185f709d26 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -54,6 +54,7 @@
#include <linux/pgtable.h>
#include <linux/uaccess.h>
+#include <asm/interrupt.h>
#include <asm/io.h>
#include <asm/irq.h>
#include <asm/cache.h>
@@ -710,7 +711,7 @@ void __do_irq(struct pt_regs *regs)
irq_exit();
}
-void do_IRQ(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_ASYNC(do_IRQ)
{
struct pt_regs *old_regs = set_irq_regs(regs);
void *cursp, *irqsp, *sirqsp;
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
index 63702c0badb9..b84459f45b1a 100644
--- a/arch/powerpc/kernel/mce.c
+++ b/arch/powerpc/kernel/mce.c
@@ -18,6 +18,7 @@
#include <linux/extable.h>
#include <linux/ftrace.h>
+#include <asm/interrupt.h>
#include <asm/machdep.h>
#include <asm/mce.h>
#include <asm/nmi.h>
@@ -588,7 +589,7 @@ EXPORT_SYMBOL_GPL(machine_check_print_event_info);
*
* regs->nip and regs->msr contains srr0 and ssr1.
*/
-long notrace machine_check_early(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early)
{
long handled = 0;
u8 ftrace_enabled = this_cpu_get_ftrace_enabled();
@@ -722,7 +723,7 @@ long hmi_handle_debugtrig(struct pt_regs *regs)
/*
* Return values:
*/
-long hmi_exception_realmode(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode)
{
int ret;
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 8e50818aa50b..15b628ae25fb 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -5,6 +5,7 @@
#include <asm/book3s/64/kup-radix.h>
#include <asm/cputime.h>
#include <asm/hw_irq.h>
+#include <asm/interrupt.h>
#include <asm/kprobes.h>
#include <asm/paca.h>
#include <asm/ptrace.h>
diff --git a/arch/powerpc/kernel/tau_6xx.c b/arch/powerpc/kernel/tau_6xx.c
index 0b4694b8d248..8ff7b9c3d839 100644
--- a/arch/powerpc/kernel/tau_6xx.c
+++ b/arch/powerpc/kernel/tau_6xx.c
@@ -100,7 +100,7 @@ static void TAUupdate(int cpu)
* with interrupts disabled
*/
-void TAUException(struct pt_regs * regs)
+DEFINE_INTERRUPT_HANDLER(TAUException) /* XXX async? */
{
int cpu = smp_processor_id();
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 74efe46f5532..bf38600a871b 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -56,6 +56,7 @@
#include <linux/processor.h>
#include <asm/trace.h>
+#include <asm/interrupt.h>
#include <asm/io.h>
#include <asm/nvram.h>
#include <asm/cache.h>
@@ -545,7 +546,7 @@ void arch_irq_work_raise(void)
* timer_interrupt - gets called when the decrementer overflows,
* with interrupts disabled.
*/
-void timer_interrupt(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt)
{
struct clock_event_device *evt = this_cpu_ptr(&decrementers);
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 902fcbd1a778..823fa827a70e 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -41,6 +41,7 @@
#include <asm/emulated_ops.h>
#include <linux/uaccess.h>
#include <asm/debugfs.h>
+#include <asm/interrupt.h>
#include <asm/io.h>
#include <asm/machdep.h>
#include <asm/rtas.h>
@@ -436,8 +437,9 @@ void hv_nmi_check_nonrecoverable(struct pt_regs *regs)
regs->msr &= ~MSR_RI;
#endif
}
+/* XXX make this nokprobe? */
-void system_reset_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception)
{
unsigned long hsrr0, hsrr1;
bool saved_hsrrs = false;
@@ -522,6 +524,8 @@ void system_reset_exception(struct pt_regs *regs)
this_cpu_set_ftrace_enabled(ftrace_enabled);
/* What should we do here? We could issue a shutdown or hard reset. */
+
+ return 0;
}
/*
@@ -819,7 +823,12 @@ int machine_check_generic(struct pt_regs *regs)
}
#endif /* everything else */
-void machine_check_exception(struct pt_regs *regs)
+
+#ifdef CONFIG_PPC_BOOK3S_64
+DEFINE_INTERRUPT_HANDLER_ASYNC(machine_check_exception)
+#else
+DEFINE_INTERRUPT_HANDLER_NMI(machine_check_exception)
+#endif
{
int recover = 0;
@@ -869,13 +878,20 @@ void machine_check_exception(struct pt_regs *regs)
if (!(regs->msr & MSR_RI))
die("Unrecoverable Machine check", regs, SIGBUS);
+#ifdef CONFIG_PPC_BOOK3S_64
+bail:
return;
+#else
+ return 0;
bail:
if (nmi) nmi_exit();
+
+ return 0;
+#endif
}
-void SMIException(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(SMIException) /* async? */
{
die("System Management Interrupt", regs, SIGABRT);
}
@@ -1061,7 +1077,7 @@ static void p9_hmi_special_emu(struct pt_regs *regs)
}
#endif /* CONFIG_VSX */
-void handle_hmi_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception)
{
struct pt_regs *old_regs;
@@ -1090,7 +1106,19 @@ void handle_hmi_exception(struct pt_regs *regs)
set_irq_regs(old_regs);
}
-void unknown_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(unknown_exception)
+{
+ enum ctx_state prev_state = exception_enter();
+
+ printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
+ regs->nip, regs->msr, regs->trap);
+
+ _exception(SIGTRAP, regs, TRAP_UNK, 0);
+
+ exception_exit(prev_state);
+}
+
+DEFINE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception)
{
enum ctx_state prev_state = exception_enter();
@@ -1102,7 +1130,7 @@ void unknown_exception(struct pt_regs *regs)
exception_exit(prev_state);
}
-void instruction_breakpoint_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception)
{
enum ctx_state prev_state = exception_enter();
@@ -1117,12 +1145,12 @@ void instruction_breakpoint_exception(struct pt_regs *regs)
exception_exit(prev_state);
}
-void RunModeException(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(RunModeException)
{
_exception(SIGTRAP, regs, TRAP_UNK, 0);
}
-void single_step_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(single_step_exception)
{
enum ctx_state prev_state = exception_enter();
@@ -1465,7 +1493,7 @@ static int emulate_math(struct pt_regs *regs)
static inline int emulate_math(struct pt_regs *regs) { return -1; }
#endif
-void program_check_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(program_check_exception)
{
enum ctx_state prev_state = exception_enter();
unsigned int reason = get_reason(regs);
@@ -1590,14 +1618,14 @@ NOKPROBE_SYMBOL(program_check_exception);
* This occurs when running in hypervisor mode on POWER6 or later
* and an illegal instruction is encountered.
*/
-void emulation_assist_interrupt(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(emulation_assist_interrupt)
{
regs->msr |= REASON_ILLEGAL;
program_check_exception(regs);
}
NOKPROBE_SYMBOL(emulation_assist_interrupt);
-void alignment_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(alignment_exception)
{
enum ctx_state prev_state = exception_enter();
int sig, code, fixed = 0;
@@ -1647,7 +1675,7 @@ void alignment_exception(struct pt_regs *regs)
exception_exit(prev_state);
}
-void StackOverflow(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(StackOverflow)
{
pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n",
current->comm, task_pid_nr(current), regs->gpr[1]);
@@ -1656,7 +1684,7 @@ void StackOverflow(struct pt_regs *regs)
panic("kernel stack overflow");
}
-void stack_overflow_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(stack_overflow_exception)
{
enum ctx_state prev_state = exception_enter();
@@ -1665,7 +1693,7 @@ void stack_overflow_exception(struct pt_regs *regs)
exception_exit(prev_state);
}
-void kernel_fp_unavailable_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception)
{
enum ctx_state prev_state = exception_enter();
@@ -1676,7 +1704,7 @@ void kernel_fp_unavailable_exception(struct pt_regs *regs)
exception_exit(prev_state);
}
-void altivec_unavailable_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(altivec_unavailable_exception)
{
enum ctx_state prev_state = exception_enter();
@@ -1695,7 +1723,7 @@ void altivec_unavailable_exception(struct pt_regs *regs)
exception_exit(prev_state);
}
-void vsx_unavailable_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(vsx_unavailable_exception)
{
if (user_mode(regs)) {
/* A user program has executed an vsx instruction,
@@ -1726,7 +1754,7 @@ static void tm_unavailable(struct pt_regs *regs)
die("Unrecoverable TM Unavailable Exception", regs, SIGABRT);
}
-void facility_unavailable_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception)
{
static char *facility_strings[] = {
[FSCR_FP_LG] = "FPU",
@@ -1846,7 +1874,7 @@ void facility_unavailable_exception(struct pt_regs *regs)
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-void fp_unavailable_tm(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(fp_unavailable_tm)
{
/* Note: This does not handle any kind of FP laziness. */
@@ -1879,7 +1907,7 @@ void fp_unavailable_tm(struct pt_regs *regs)
tm_recheckpoint(¤t->thread);
}
-void altivec_unavailable_tm(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(altivec_unavailable_tm)
{
/* See the comments in fp_unavailable_tm(). This function operates
* the same way.
@@ -1894,7 +1922,7 @@ void altivec_unavailable_tm(struct pt_regs *regs)
current->thread.used_vr = 1;
}
-void vsx_unavailable_tm(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(vsx_unavailable_tm)
{
/* See the comments in fp_unavailable_tm(). This works similarly,
* though we're loading both FP and VEC registers in here.
@@ -1919,7 +1947,7 @@ void vsx_unavailable_tm(struct pt_regs *regs)
}
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
-void performance_monitor_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception)
{
__this_cpu_inc(irq_stat.pmu_irqs);
@@ -2057,7 +2085,7 @@ NOKPROBE_SYMBOL(DebugException);
#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
#ifdef CONFIG_ALTIVEC
-void altivec_assist_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(altivec_assist_exception)
{
int err;
@@ -2199,7 +2227,7 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)
* in the MSR is 0. This indicates that SRR0/1 are live, and that
* we therefore lost state by taking this exception.
*/
-void unrecoverable_exception(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(unrecoverable_exception)
{
pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n",
regs->trap, regs->nip, regs->msr);
@@ -2219,7 +2247,7 @@ void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs)
return;
}
-void WatchdogException(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(WatchdogException) /* XXX NMI? async? */
{
printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n");
WatchdogHandler(regs);
@@ -2230,7 +2258,7 @@ void WatchdogException(struct pt_regs *regs)
* We enter here if we discover during exception entry that we are
* running in supervisor mode with a userspace value in the stack pointer.
*/
-void kernel_bad_stack(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(kernel_bad_stack)
{
printk(KERN_EMERG "Bad kernel stack pointer %lx at %lx\n",
regs->gpr[1], regs->nip);
diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c
index af3c15a1d41e..824b9376ac35 100644
--- a/arch/powerpc/kernel/watchdog.c
+++ b/arch/powerpc/kernel/watchdog.c
@@ -26,6 +26,7 @@
#include <linux/delay.h>
#include <linux/smp.h>
+#include <asm/interrupt.h>
#include <asm/paca.h>
/*
@@ -247,14 +248,14 @@ static void watchdog_timer_interrupt(int cpu)
watchdog_smp_panic(cpu, tb);
}
-void soft_nmi_interrupt(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)
{
unsigned long flags;
int cpu = raw_smp_processor_id();
u64 tb;
if (!cpumask_test_cpu(cpu, &wd_cpus_enabled))
- return;
+ return 0;
nmi_enter();
@@ -291,6 +292,8 @@ void soft_nmi_interrupt(struct pt_regs *regs)
out:
nmi_exit();
+
+ return 0;
}
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index e3b1839fc251..01816de0e0ec 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -53,6 +53,7 @@
#include <asm/cputable.h>
#include <asm/cacheflush.h>
#include <linux/uaccess.h>
+#include <asm/interrupt.h>
#include <asm/io.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index 8f58dd20b362..a2f3e6e70361 100644
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -17,6 +17,7 @@
#include <asm/asm-prototypes.h>
#include <asm/cputable.h>
+#include <asm/interrupt.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <asm/archrandom.h>
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index 731518e7d56f..a48c484b9e9b 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -38,6 +38,7 @@
#include <linux/pgtable.h>
#include <asm/debugfs.h>
+#include <asm/interrupt.h>
#include <asm/processor.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
@@ -1510,7 +1511,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
}
EXPORT_SYMBOL_GPL(hash_page);
-long do_hash_fault(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_RET(do_hash_fault)
{
unsigned long ea = regs->dar;
unsigned long dsisr = regs->dsisr;
diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c
index ae89ad516247..efac69e73ca8 100644
--- a/arch/powerpc/mm/book3s64/slb.c
+++ b/arch/powerpc/mm/book3s64/slb.c
@@ -10,6 +10,7 @@
*/
#include <asm/asm-prototypes.h>
+#include <asm/interrupt.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
#include <asm/paca.h>
@@ -837,7 +838,7 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)
return slb_insert_entry(ea, context, flags, ssize, false);
}
-long do_slb_fault(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_RAW(do_slb_fault)
{
unsigned long ea = regs->dar;
unsigned long id = get_region_id(ea);
@@ -890,7 +891,7 @@ long do_slb_fault(struct pt_regs *regs)
}
}
-void do_bad_slb_fault(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER(do_bad_slb_fault)
{
int err = regs->result;
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 49fbe564ea2b..d1c5ef853860 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -34,6 +34,7 @@
#include <linux/uaccess.h>
#include <asm/firmware.h>
+#include <asm/interrupt.h>
#include <asm/page.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
@@ -549,7 +550,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
}
NOKPROBE_SYMBOL(__do_page_fault);
-long do_page_fault(struct pt_regs *regs)
+DEFINE_INTERRUPT_HANDLER_RET(do_page_fault)
{
enum ctx_state prev_state = exception_enter();
unsigned long address = regs->dar;
@@ -656,3 +657,10 @@ void bad_page_fault(struct pt_regs *regs, int sig)
die("Kernel access of bad area", regs, sig);
}
+
+#ifdef CONFIG_PPC_BOOK3S_64
+DEFINE_INTERRUPT_HANDLER(do_bad_page_fault)
+{
+ bad_page_fault(regs, SIGSEGV);
+}
+#endif
diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c
index 1ed7c5286487..2fc7049fef53 100644
--- a/arch/powerpc/platforms/powernv/idle.c
+++ b/arch/powerpc/platforms/powernv/idle.c
@@ -14,6 +14,7 @@
#include <asm/asm-prototypes.h>
#include <asm/firmware.h>
+#include <asm/interrupt.h>
#include <asm/machdep.h>
#include <asm/opal.h>
#include <asm/cputhreads.h>
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 05/18] powerpc: add interrupt wrapper entry / exit stub functions
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (3 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 04/18] powerpc: interrupt handler wrapper functions Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 06/18] powerpc: add interrupt_cond_local_irq_enable helper Nicholas Piggin
` (12 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
These will be used by subsequent patches.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 52 ++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 3292f5094085..c5ffcf144bbd 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -5,6 +5,36 @@
#include <linux/context_tracking.h>
#include <asm/ftrace.h>
+struct interrupt_state {
+};
+
+static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
+{
+}
+
+static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
+{
+}
+
+static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
+{
+}
+
+static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
+{
+}
+
+struct interrupt_nmi_state {
+};
+
+static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
+{
+}
+
+static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
+{
+}
+
/**
* DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function
* @func: Function name of the entry point
@@ -59,7 +89,13 @@ static __always_inline void ___##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
+ struct interrupt_state state; \
+ \
+ interrupt_enter_prepare(regs, &state); \
+ \
___##func (regs); \
+ \
+ interrupt_exit_prepare(regs, &state); \
} \
\
static __always_inline void ___##func(struct pt_regs *regs)
@@ -87,10 +123,15 @@ static __always_inline long ___##func(struct pt_regs *regs); \
\
__visible noinstr long func(struct pt_regs *regs) \
{ \
+ struct interrupt_state state; \
long ret; \
\
+ interrupt_enter_prepare(regs, &state); \
+ \
ret = ___##func (regs); \
\
+ interrupt_exit_prepare(regs, &state); \
+ \
return ret; \
} \
\
@@ -117,7 +158,13 @@ static __always_inline void ___##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
+ struct interrupt_state state; \
+ \
+ interrupt_async_enter_prepare(regs, &state); \
+ \
___##func (regs); \
+ \
+ interrupt_async_exit_prepare(regs, &state); \
} \
\
static __always_inline void ___##func(struct pt_regs *regs)
@@ -145,10 +192,15 @@ static __always_inline long ___##func(struct pt_regs *regs); \
\
__visible noinstr long func(struct pt_regs *regs) \
{ \
+ struct interrupt_nmi_state state; \
long ret; \
\
+ interrupt_nmi_enter_prepare(regs, &state); \
+ \
ret = ___##func (regs); \
\
+ interrupt_nmi_exit_prepare(regs, &state); \
+ \
return ret; \
} \
\
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 06/18] powerpc: add interrupt_cond_local_irq_enable helper
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (4 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 05/18] powerpc: add interrupt wrapper entry / exit stub functions Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 07/18] powerpc/64: context tracking remove _TIF_NOHZ Nicholas Piggin
` (11 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Simple helper for synchronous interrupt handlers to use to enable
interrupts if they were taken in interrupt-enabled context.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 7 +++++++
arch/powerpc/kernel/traps.c | 24 +++++++-----------------
arch/powerpc/mm/fault.c | 4 +---
3 files changed, 15 insertions(+), 20 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index c5ffcf144bbd..446e24b0eee1 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -3,6 +3,7 @@
#define _ASM_POWERPC_INTERRUPT_H
#include <linux/context_tracking.h>
+#include <linux/hardirq.h>
#include <asm/ftrace.h>
struct interrupt_state {
@@ -251,4 +252,10 @@ DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
void replay_system_reset(void);
void replay_soft_interrupts(void);
+static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs)
+{
+ if (!arch_irq_disabled_regs(regs))
+ local_irq_enable();
+}
+
#endif /* _ASM_POWERPC_INTERRUPT_H */
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 823fa827a70e..2d19df1f6c05 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -343,8 +343,8 @@ static bool exception_common(int signr, struct pt_regs *regs, int code,
show_signal_msg(signr, regs, code, addr);
- if (arch_irqs_disabled() && !arch_irq_disabled_regs(regs))
- local_irq_enable();
+ if (arch_irqs_disabled())
+ interrupt_cond_local_irq_enable(regs);
current->thread.trap_nr = code;
@@ -1575,9 +1575,7 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
if (!user_mode(regs))
goto sigill;
- /* We restore the interrupt state now */
- if (!arch_irq_disabled_regs(regs))
- local_irq_enable();
+ interrupt_cond_local_irq_enable(regs);
/* (reason & REASON_ILLEGAL) would be the obvious thing here,
* but there seems to be a hardware bug on the 405GP (RevD)
@@ -1631,9 +1629,7 @@ DEFINE_INTERRUPT_HANDLER(alignment_exception)
int sig, code, fixed = 0;
unsigned long reason;
- /* We restore the interrupt state now */
- if (!arch_irq_disabled_regs(regs))
- local_irq_enable();
+ interrupt_cond_local_irq_enable(regs);
reason = get_reason(regs);
@@ -1794,9 +1790,7 @@ DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception)
die("Unexpected facility unavailable exception", regs, SIGABRT);
}
- /* We restore the interrupt state now */
- if (!arch_irq_disabled_regs(regs))
- local_irq_enable();
+ interrupt_cond_local_irq_enable(regs);
if (status == FSCR_DSCR_LG) {
/*
@@ -2141,9 +2135,7 @@ void SPEFloatingPointException(struct pt_regs *regs)
int code = FPE_FLTUNK;
int err;
- /* We restore the interrupt state now */
- if (!arch_irq_disabled_regs(regs))
- local_irq_enable();
+ interrupt_cond_local_irq_enable(regs);
flush_spe_to_thread(current);
@@ -2190,9 +2182,7 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)
extern int speround_handler(struct pt_regs *regs);
int err;
- /* We restore the interrupt state now */
- if (!arch_irq_disabled_regs(regs))
- local_irq_enable();
+ interrupt_cond_local_irq_enable(regs);
preempt_disable();
if (regs->msr & MSR_SPE)
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index d1c5ef853860..fd0c56c80c3c 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -443,9 +443,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
return bad_area_nosemaphore(regs, address);
}
- /* We restore the interrupt state now */
- if (!arch_irq_disabled_regs(regs))
- local_irq_enable();
+ interrupt_cond_local_irq_enable(regs);
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 07/18] powerpc/64: context tracking remove _TIF_NOHZ
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (5 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 06/18] powerpc: add interrupt_cond_local_irq_enable helper Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 08/18] powerpc/64: context tracking move to interrupt wrappers Nicholas Piggin
` (10 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Add context tracking to the system call handler explicitly, and remove
_TIF_NOHZ.
This saves 35 cycles on gettid system call cost on POWER9 with a
CONFIG_NOHZ_FULL kernel.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/Kconfig | 6 ------
arch/powerpc/Kconfig | 1 -
arch/powerpc/include/asm/thread_info.h | 4 +---
arch/powerpc/kernel/ptrace/ptrace.c | 4 ----
arch/powerpc/kernel/signal.c | 4 ----
arch/powerpc/kernel/syscall_64.c | 10 ++++++++++
6 files changed, 11 insertions(+), 18 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 56b6ccc0e32d..a0b6213f7820 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -618,12 +618,6 @@ config HAVE_CONTEXT_TRACKING
protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal
handling on irq exit still need to be protected.
-config HAVE_TIF_NOHZ
- bool
- help
- Arch relies on TIF_NOHZ and syscall slow path to implement context
- tracking calls to user_enter()/user_exit().
-
config HAVE_VIRT_CPU_ACCOUNTING
bool
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index e9f13fe08492..6eaf12a504f8 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -193,7 +193,6 @@ config PPC
select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2)
select HAVE_CONTEXT_TRACKING if PPC64
- select HAVE_TIF_NOHZ if PPC64
select HAVE_DEBUG_KMEMLEAK
select HAVE_DEBUG_STACKOVERFLOW
select HAVE_DYNAMIC_FTRACE
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index 46a210b03d2b..c9443c16e5fb 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -95,7 +95,6 @@ void arch_setup_new_exec(void);
#define TIF_PATCH_PENDING 6 /* pending live patching update */
#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
#define TIF_SINGLESTEP 8 /* singlestepping active */
-#define TIF_NOHZ 9 /* in adaptive nohz mode */
#define TIF_SECCOMP 10 /* secure computing */
#define TIF_RESTOREALL 11 /* Restore all regs (implies NOERROR) */
#define TIF_NOERROR 12 /* Force successful syscall return */
@@ -128,11 +127,10 @@ void arch_setup_new_exec(void);
#define _TIF_UPROBE (1<<TIF_UPROBE)
#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
#define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE)
-#define _TIF_NOHZ (1<<TIF_NOHZ)
#define _TIF_SYSCALL_EMU (1<<TIF_SYSCALL_EMU)
#define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \
- _TIF_NOHZ | _TIF_SYSCALL_EMU)
+ _TIF_SYSCALL_EMU)
#define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
_TIF_NOTIFY_RESUME | _TIF_UPROBE | \
diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c
index f6e51be47c6e..8970400e521c 100644
--- a/arch/powerpc/kernel/ptrace/ptrace.c
+++ b/arch/powerpc/kernel/ptrace/ptrace.c
@@ -290,8 +290,6 @@ long do_syscall_trace_enter(struct pt_regs *regs)
{
u32 flags;
- user_exit();
-
flags = READ_ONCE(current_thread_info()->flags) &
(_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE);
@@ -368,8 +366,6 @@ void do_syscall_trace_leave(struct pt_regs *regs)
step = test_thread_flag(TIF_SINGLESTEP);
if (step || test_thread_flag(TIF_SYSCALL_TRACE))
tracehook_report_syscall_exit(regs, step);
-
- user_enter();
}
void __init pt_regs_check(void);
diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c
index d2c356f37077..44ec7b34b27e 100644
--- a/arch/powerpc/kernel/signal.c
+++ b/arch/powerpc/kernel/signal.c
@@ -310,8 +310,6 @@ static void do_signal(struct task_struct *tsk)
void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
{
- user_exit();
-
if (thread_info_flags & _TIF_UPROBE)
uprobe_notify_resume(regs);
@@ -327,8 +325,6 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
tracehook_notify_resume(regs);
rseq_handle_notify_resume(NULL, regs);
}
-
- user_enter();
}
unsigned long get_tm_stackpointer(struct task_struct *tsk)
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 15b628ae25fb..d9df6d14533e 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -1,9 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/context_tracking.h>
#include <linux/err.h>
#include <asm/asm-prototypes.h>
#include <asm/book3s/64/kup-radix.h>
#include <asm/cputime.h>
+#include <asm/interrupt.h>
#include <asm/hw_irq.h>
#include <asm/interrupt.h>
#include <asm/kprobes.h>
@@ -28,6 +30,9 @@ notrace long system_call_exception(long r3, long r4, long r5,
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
+ CT_WARN_ON(ct_state() == CONTEXT_KERNEL);
+ user_exit_irqoff();
+
trace_hardirqs_off(); /* finish reconciling */
if (IS_ENABLED(CONFIG_PPC_BOOK3S))
@@ -158,6 +163,8 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
unsigned long ti_flags;
unsigned long ret = 0;
+ CT_WARN_ON(ct_state() == CONTEXT_USER);
+
kuap_check_amr();
regs->result = r3;
@@ -234,8 +241,11 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
}
}
+ user_enter_irqoff();
+
/* scv need not set RI=0 because SRRs are not used */
if (unlikely(!prep_irq_for_enabled_exit(!scv))) {
+ user_exit_irqoff();
local_irq_enable();
goto again;
}
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 08/18] powerpc/64: context tracking move to interrupt wrappers
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (6 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 07/18] powerpc/64: context tracking remove _TIF_NOHZ Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 09/18] powerpc/64: add context tracking to asynchronous interrupts Nicholas Piggin
` (9 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
This moves exception_enter/exit calls to wrapper functions for
synchronous interrupts. More interrupt handlers are covered by
this than previously.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 9 ++++
arch/powerpc/kernel/traps.c | 74 ++++++---------------------
arch/powerpc/mm/book3s64/hash_utils.c | 2 -
arch/powerpc/mm/fault.c | 3 --
4 files changed, 26 insertions(+), 62 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 446e24b0eee1..62a362915adc 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -7,14 +7,23 @@
#include <asm/ftrace.h>
struct interrupt_state {
+#ifdef CONFIG_PPC64
+ enum ctx_state ctx_state;
+#endif
};
static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
+#ifdef CONFIG_PPC64
+ state->ctx_state = exception_enter();
+#endif
}
static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
+#ifdef CONFIG_PPC64
+ exception_exit(state->ctx_state);
+#endif
}
static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 2d19df1f6c05..09780481e1b1 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1108,41 +1108,28 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception)
DEFINE_INTERRUPT_HANDLER(unknown_exception)
{
- enum ctx_state prev_state = exception_enter();
-
printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
regs->nip, regs->msr, regs->trap);
_exception(SIGTRAP, regs, TRAP_UNK, 0);
-
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception)
{
- enum ctx_state prev_state = exception_enter();
-
printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
regs->nip, regs->msr, regs->trap);
_exception(SIGTRAP, regs, TRAP_UNK, 0);
-
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception)
{
- enum ctx_state prev_state = exception_enter();
-
if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5,
5, SIGTRAP) == NOTIFY_STOP)
- goto bail;
+ return;
if (debugger_iabr_match(regs))
- goto bail;
+ return;
_exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
-
-bail:
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER(RunModeException)
@@ -1152,8 +1139,6 @@ DEFINE_INTERRUPT_HANDLER(RunModeException)
DEFINE_INTERRUPT_HANDLER(single_step_exception)
{
- enum ctx_state prev_state = exception_enter();
-
clear_single_step(regs);
clear_br_trace(regs);
@@ -1162,14 +1147,11 @@ DEFINE_INTERRUPT_HANDLER(single_step_exception)
if (notify_die(DIE_SSTEP, "single_step", regs, 5,
5, SIGTRAP) == NOTIFY_STOP)
- goto bail;
+ return;
if (debugger_sstep(regs))
- goto bail;
+ return;
_exception(SIGTRAP, regs, TRAP_TRACE, regs->nip);
-
-bail:
- exception_exit(prev_state);
}
NOKPROBE_SYMBOL(single_step_exception);
@@ -1495,7 +1477,6 @@ static inline int emulate_math(struct pt_regs *regs) { return -1; }
DEFINE_INTERRUPT_HANDLER(program_check_exception)
{
- enum ctx_state prev_state = exception_enter();
unsigned int reason = get_reason(regs);
/* We can now get here via a FP Unavailable exception if the core
@@ -1504,22 +1485,22 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
if (reason & REASON_FP) {
/* IEEE FP exception */
parse_fpe(regs);
- goto bail;
+ return;
}
if (reason & REASON_TRAP) {
unsigned long bugaddr;
/* Debugger is first in line to stop recursive faults in
* rcu_lock, notify_die, or atomic_notifier_call_chain */
if (debugger_bpt(regs))
- goto bail;
+ return;
if (kprobe_handler(regs))
- goto bail;
+ return;
/* trap exception */
if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP)
== NOTIFY_STOP)
- goto bail;
+ return;
bugaddr = regs->nip;
/*
@@ -1531,10 +1512,10 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
if (!(regs->msr & MSR_PR) && /* not user-mode */
report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) {
regs->nip += 4;
- goto bail;
+ return;
}
_exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
- goto bail;
+ return;
}
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
if (reason & REASON_TM) {
@@ -1555,7 +1536,7 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
*/
if (user_mode(regs)) {
_exception(SIGILL, regs, ILL_ILLOPN, regs->nip);
- goto bail;
+ return;
} else {
printk(KERN_EMERG "Unexpected TM Bad Thing exception "
"at %lx (msr 0x%lx) tm_scratch=%llx\n",
@@ -1586,7 +1567,7 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
* pattern to occurrences etc. -dgibson 31/Mar/2003
*/
if (!emulate_math(regs))
- goto bail;
+ return;
/* Try to emulate it if we should. */
if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) {
@@ -1594,10 +1575,10 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
case 0:
regs->nip += 4;
emulate_single_step(regs);
- goto bail;
+ return;
case -EFAULT:
_exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip);
- goto bail;
+ return;
}
}
@@ -1606,9 +1587,6 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
_exception(SIGILL, regs, ILL_PRVOPC, regs->nip);
else
_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
-
-bail:
- exception_exit(prev_state);
}
NOKPROBE_SYMBOL(program_check_exception);
@@ -1625,14 +1603,12 @@ NOKPROBE_SYMBOL(emulation_assist_interrupt);
DEFINE_INTERRUPT_HANDLER(alignment_exception)
{
- enum ctx_state prev_state = exception_enter();
int sig, code, fixed = 0;
unsigned long reason;
interrupt_cond_local_irq_enable(regs);
reason = get_reason(regs);
-
if (reason & REASON_BOUNDARY) {
sig = SIGBUS;
code = BUS_ADRALN;
@@ -1640,7 +1616,7 @@ DEFINE_INTERRUPT_HANDLER(alignment_exception)
}
if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT))
- goto bail;
+ return;
/* we don't implement logging of alignment exceptions */
if (!(current->thread.align_ctl & PR_UNALIGN_SIGBUS))
@@ -1650,7 +1626,7 @@ DEFINE_INTERRUPT_HANDLER(alignment_exception)
/* skip over emulated instruction */
regs->nip += inst_length(reason);
emulate_single_step(regs);
- goto bail;
+ return;
}
/* Operand address was bad */
@@ -1666,9 +1642,6 @@ DEFINE_INTERRUPT_HANDLER(alignment_exception)
_exception(sig, regs, code, regs->dar);
else
bad_page_fault(regs, sig);
-
-bail:
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER(StackOverflow)
@@ -1682,41 +1655,28 @@ DEFINE_INTERRUPT_HANDLER(StackOverflow)
DEFINE_INTERRUPT_HANDLER(stack_overflow_exception)
{
- enum ctx_state prev_state = exception_enter();
-
die("Kernel stack overflow", regs, SIGSEGV);
-
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception)
{
- enum ctx_state prev_state = exception_enter();
-
printk(KERN_EMERG "Unrecoverable FP Unavailable Exception "
"%lx at %lx\n", regs->trap, regs->nip);
die("Unrecoverable FP Unavailable Exception", regs, SIGABRT);
-
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER(altivec_unavailable_exception)
{
- enum ctx_state prev_state = exception_enter();
-
if (user_mode(regs)) {
/* A user program has executed an altivec instruction,
but this kernel doesn't support altivec. */
_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
- goto bail;
+ return;
}
printk(KERN_EMERG "Unrecoverable VMX/Altivec Unavailable Exception "
"%lx at %lx\n", regs->trap, regs->nip);
die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT);
-
-bail:
- exception_exit(prev_state);
}
DEFINE_INTERRUPT_HANDLER(vsx_unavailable_exception)
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index a48c484b9e9b..39dcfaf7ba36 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -1289,7 +1289,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
unsigned long flags)
{
bool is_thp;
- enum ctx_state prev_state = exception_enter();
pgd_t *pgdir;
unsigned long vsid;
pte_t *ptep;
@@ -1489,7 +1488,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
DBG_LOW(" -> rc=%d\n", rc);
bail:
- exception_exit(prev_state);
return rc;
}
EXPORT_SYMBOL_GPL(hash_page_mm);
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index fd0c56c80c3c..4cc265e0364d 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -550,7 +550,6 @@ NOKPROBE_SYMBOL(__do_page_fault);
DEFINE_INTERRUPT_HANDLER_RET(do_page_fault)
{
- enum ctx_state prev_state = exception_enter();
unsigned long address = regs->dar;
unsigned long error_code = regs->dsisr;
long err;
@@ -573,8 +572,6 @@ DEFINE_INTERRUPT_HANDLER_RET(do_page_fault)
}
#endif
- exception_exit(prev_state);
-
return err;
}
NOKPROBE_SYMBOL(do_page_fault);
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 09/18] powerpc/64: add context tracking to asynchronous interrupts
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (7 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 08/18] powerpc/64: context tracking move to interrupt wrappers Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 10/18] powerpc/64s: move context tracking exit to interrupt exit path Nicholas Piggin
` (8 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Previously context tracking was not done for asynchronous interrupts,
(those that run in interrupt context), and if those would cause a
reschedule when they exit, then scheduling functions (schedule_user,
preempt_schedule_irq) call exception_enter/exit to fix this up and
exit user context.
This is a hack we would like to get away from, so do context tracking
for asynchronous interrupts too.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 62a362915adc..55f544dbf434 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -28,10 +28,12 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt
static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
+ interrupt_enter_prepare(regs, state);
}
static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
+ interrupt_exit_prepare(regs, state);
}
struct interrupt_nmi_state {
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 10/18] powerpc/64s: move context tracking exit to interrupt exit path
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (8 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 09/18] powerpc/64: add context tracking to asynchronous interrupts Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 11/18] powerpc/64s: reconcile interrupts in C Nicholas Piggin
` (7 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
The interrupt handler wrapper functions are not the ideal place to
maintain context tracking because after they return, the low level exit
code must then determine if there are interrupts to replay, or if the
task should be preempted, etc. Those paths (e.g., schedule_user) include
their own exception_enter/exit pairs to fix this up but it's a bit hacky
(see schedule_user() comments).
Ideally context tracking will go to user mode only when there are no
more interrupts or context switches or other exit processing work to
handle.
64e can not do this because it does not use the C interrupt exit code.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 27 ++++++++++++++++++++++++---
arch/powerpc/kernel/syscall_64.c | 9 +++++++++
2 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 55f544dbf434..0b29d363988d 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -7,23 +7,44 @@
#include <asm/ftrace.h>
struct interrupt_state {
-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3E_64
enum ctx_state ctx_state;
#endif
};
static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3E_64
state->ctx_state = exception_enter();
#endif
+
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (user_mode(regs)) {
+ CT_WARN_ON(ct_state() != CONTEXT_USER);
+ user_exit_irqoff();
+ } else {
+ /*
+ * CT_WARN_ON comes here via program_check_exception,
+ * so avoid recursion.
+ */
+ if (TRAP(regs) != 0x700)
+ CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
+ }
+#endif
}
static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3E_64
exception_exit(state->ctx_state);
#endif
+
+ /*
+ * Book3S exits to user via interrupt_exit_user_prepare(), which does
+ * context tracking, which is a cleaner way to handle PREEMPT=y
+ * and avoid context entry/exit in e.g., preempt_schedule_irq()),
+ * which is likely to be where the core code wants to end up.
+ */
}
static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index d9df6d14533e..5820a18672bc 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -275,6 +275,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
BUG_ON(!(regs->msr & MSR_PR));
BUG_ON(!FULL_REGS(regs));
BUG_ON(regs->softe != IRQS_ENABLED);
+ CT_WARN_ON(ct_state() == CONTEXT_USER);
/*
* We don't need to restore AMR on the way back to userspace for KUAP.
@@ -317,7 +318,9 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
}
}
+ user_enter_irqoff();
if (unlikely(!prep_irq_for_enabled_exit(true))) {
+ user_exit_irqoff();
local_irq_enable();
local_irq_disable();
goto again;
@@ -358,6 +361,12 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
unrecoverable_exception(regs);
BUG_ON(regs->msr & MSR_PR);
BUG_ON(!FULL_REGS(regs));
+ /*
+ * CT_WARN_ON comes here via program_check_exception,
+ * so avoid recursion.
+ */
+ if (TRAP(regs) != 0x700)
+ CT_WARN_ON(ct_state() == CONTEXT_USER);
amr = kuap_get_and_check_amr();
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 11/18] powerpc/64s: reconcile interrupts in C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (9 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 10/18] powerpc/64s: move context tracking exit to interrupt exit path Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 12/18] powerpc/64: move account_stolen_time into its own function Nicholas Piggin
` (6 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
There is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 15 +++++++++++----
arch/powerpc/kernel/exceptions-64s.S | 26 --------------------------
2 files changed, 11 insertions(+), 30 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 0b29d363988d..bb09bcb290af 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -14,11 +14,14 @@ struct interrupt_state {
static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
-#ifdef CONFIG_PPC_BOOK3E_64
- state->ctx_state = exception_enter();
-#endif
-
+ /*
+ * Book3E reconciles irq soft mask in asm
+ */
#ifdef CONFIG_PPC_BOOK3S_64
+ if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED)
+ trace_hardirqs_off();
+ local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
+
if (user_mode(regs)) {
CT_WARN_ON(ct_state() != CONTEXT_USER);
user_exit_irqoff();
@@ -31,6 +34,10 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
}
#endif
+
+#ifdef CONFIG_PPC_BOOK3E_64
+ state->ctx_state = exception_enter();
+#endif
}
static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index f6989321136d..b36247ad1f64 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -139,7 +139,6 @@ name:
#define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */
#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */
#define __ISTACK(name) .L_ISTACK_ ## name
-#define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */
#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */
#define INT_DEFINE_BEGIN(n) \
@@ -203,9 +202,6 @@ do_define_int n
.ifndef ISTACK
ISTACK=1
.endif
- .ifndef IRECONCILE
- IRECONCILE=1
- .endif
.ifndef IKUAP
IKUAP=1
.endif
@@ -653,10 +649,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
.if ISTACK
ACCOUNT_STOLEN_TIME
.endif
-
- .if IRECONCILE
- RECONCILE_IRQ_STATE(r10, r11)
- .endif
.endm
/*
@@ -935,7 +927,6 @@ INT_DEFINE_BEGIN(system_reset)
*/
ISET_RI=0
ISTACK=0
- IRECONCILE=0
IKVM_REAL=1
INT_DEFINE_END(system_reset)
@@ -1125,7 +1116,6 @@ INT_DEFINE_BEGIN(machine_check_early)
ISTACK=0
IDAR=1
IDSISR=1
- IRECONCILE=0
IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
INT_DEFINE_END(machine_check_early)
@@ -1473,7 +1463,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
INT_DEFINE_BEGIN(data_access_slb)
IVEC=0x380
IAREA=PACA_EXSLB
- IRECONCILE=0
IDAR=1
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_SKIP=1
@@ -1502,7 +1491,6 @@ MMU_FTR_SECTION_ELSE
li r3,-EFAULT
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
std r3,RESULT(r1)
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_bad_slb_fault
b interrupt_return
@@ -1564,7 +1552,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
INT_DEFINE_BEGIN(instruction_access_slb)
IVEC=0x480
IAREA=PACA_EXSLB
- IRECONCILE=0
IISIDE=1
IDAR=1
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
@@ -1593,7 +1580,6 @@ MMU_FTR_SECTION_ELSE
li r3,-EFAULT
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
std r3,RESULT(r1)
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_bad_slb_fault
b interrupt_return
@@ -1753,7 +1739,6 @@ EXC_COMMON_BEGIN(program_check_common)
*/
INT_DEFINE_BEGIN(fp_unavailable)
IVEC=0x800
- IRECONCILE=0
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
@@ -1768,7 +1753,6 @@ EXC_VIRT_END(fp_unavailable, 0x4800, 0x100)
EXC_COMMON_BEGIN(fp_unavailable_common)
GEN_COMMON fp_unavailable
bne 1f /* if from user, just load it up */
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl kernel_fp_unavailable_exception
0: trap
@@ -1787,7 +1771,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM)
b fast_interrupt_return
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2: /* User process was in a transaction */
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl fp_unavailable_tm
b interrupt_return
@@ -1852,7 +1835,6 @@ INT_DEFINE_BEGIN(hdecrementer)
IVEC=0x980
IHSRR=1
ISTACK=0
- IRECONCILE=0
IKVM_REAL=1
IKVM_VIRT=1
INT_DEFINE_END(hdecrementer)
@@ -2226,7 +2208,6 @@ INT_DEFINE_BEGIN(hmi_exception_early)
IHSRR=1
IREALMODE_COMMON=1
ISTACK=0
- IRECONCILE=0
IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
IKVM_REAL=1
INT_DEFINE_END(hmi_exception_early)
@@ -2400,7 +2381,6 @@ EXC_COMMON_BEGIN(performance_monitor_common)
*/
INT_DEFINE_BEGIN(altivec_unavailable)
IVEC=0xf20
- IRECONCILE=0
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
@@ -2430,7 +2410,6 @@ BEGIN_FTR_SECTION
b fast_interrupt_return
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2: /* User process was in a transaction */
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl altivec_unavailable_tm
b interrupt_return
@@ -2438,7 +2417,6 @@ BEGIN_FTR_SECTION
1:
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl altivec_unavailable_exception
b interrupt_return
@@ -2454,7 +2432,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
*/
INT_DEFINE_BEGIN(vsx_unavailable)
IVEC=0xf40
- IRECONCILE=0
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
@@ -2483,7 +2460,6 @@ BEGIN_FTR_SECTION
b load_up_vsx
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2: /* User process was in a transaction */
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl vsx_unavailable_tm
b interrupt_return
@@ -2491,7 +2467,6 @@ BEGIN_FTR_SECTION
1:
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
- RECONCILE_IRQ_STATE(r10, r11)
addi r3,r1,STACK_FRAME_OVERHEAD
bl vsx_unavailable_exception
b interrupt_return
@@ -2826,7 +2801,6 @@ EXC_VIRT_NONE(0x5800, 0x100)
INT_DEFINE_BEGIN(soft_nmi)
IVEC=0x900
ISTACK=0
- IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */
INT_DEFINE_END(soft_nmi)
/*
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 12/18] powerpc/64: move account_stolen_time into its own function
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (10 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 11/18] powerpc/64s: reconcile interrupts in C Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 13/18] powerpc/64: entry cpu time accounting in C Nicholas Piggin
` (5 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
This will be used by interrupt entry as well.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/cputime.h | 15 +++++++++++++++
arch/powerpc/kernel/syscall_64.c | 10 +---------
2 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h
index ed75d1c318e3..3f61604e1fcf 100644
--- a/arch/powerpc/include/asm/cputime.h
+++ b/arch/powerpc/include/asm/cputime.h
@@ -87,6 +87,18 @@ static notrace inline void account_cpu_user_exit(void)
acct->starttime_user = tb;
}
+static notrace inline void account_stolen_time(void)
+{
+#ifdef CONFIG_PPC_SPLPAR
+ if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) &&
+ firmware_has_feature(FW_FEATURE_SPLPAR)) {
+ struct lppaca *lp = local_paca->lppaca_ptr;
+
+ if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx)))
+ accumulate_stolen_time();
+ }
+#endif
+}
#endif /* __KERNEL__ */
#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
@@ -96,5 +108,8 @@ static inline void account_cpu_user_entry(void)
static inline void account_cpu_user_exit(void)
{
}
+static notrace inline void account_stolen_time(void)
+{
+}
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
#endif /* __POWERPC_CPUTIME_H */
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 5820a18672bc..672f2a796487 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -45,15 +45,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
account_cpu_user_entry();
-#ifdef CONFIG_PPC_SPLPAR
- if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) &&
- firmware_has_feature(FW_FEATURE_SPLPAR)) {
- struct lppaca *lp = local_paca->lppaca_ptr;
-
- if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx)))
- accumulate_stolen_time();
- }
-#endif
+ account_stolen_time();
/*
* This is not required for the syscall exit path, but makes the
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 13/18] powerpc/64: entry cpu time accounting in C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (11 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 12/18] powerpc/64: move account_stolen_time into its own function Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 14/18] powerpc: move NMI entry/exit code into wrapper Nicholas Piggin
` (4 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
There is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 4 ++++
arch/powerpc/include/asm/ppc_asm.h | 24 ------------------------
arch/powerpc/kernel/exceptions-64e.S | 1 -
arch/powerpc/kernel/exceptions-64s.S | 5 -----
4 files changed, 4 insertions(+), 30 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index bb09bcb290af..9bd14c59dc8e 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -4,6 +4,7 @@
#include <linux/context_tracking.h>
#include <linux/hardirq.h>
+#include <asm/cputime.h>
#include <asm/ftrace.h>
struct interrupt_state {
@@ -25,6 +26,9 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
if (user_mode(regs)) {
CT_WARN_ON(ct_state() != CONTEXT_USER);
user_exit_irqoff();
+
+ account_cpu_user_entry();
+ account_stolen_time();
} else {
/*
* CT_WARN_ON comes here via program_check_exception,
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index 511786f0e40d..c1199f6c75a3 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -25,7 +25,6 @@
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
#define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb)
#define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb)
-#define ACCOUNT_STOLEN_TIME
#else
#define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) \
MFTB(ra); /* get timebase */ \
@@ -44,29 +43,6 @@
PPC_LL ra, ACCOUNT_SYSTEM_TIME(ptr); \
add ra,ra,rb; /* add on to system time */ \
PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr)
-
-#ifdef CONFIG_PPC_SPLPAR
-#define ACCOUNT_STOLEN_TIME \
-BEGIN_FW_FTR_SECTION; \
- beq 33f; \
- /* from user - see if there are any DTL entries to process */ \
- ld r10,PACALPPACAPTR(r13); /* get ptr to VPA */ \
- ld r11,PACA_DTL_RIDX(r13); /* get log read index */ \
- addi r10,r10,LPPACA_DTLIDX; \
- LDX_BE r10,0,r10; /* get log write index */ \
- cmpd cr1,r11,r10; \
- beq+ cr1,33f; \
- bl accumulate_stolen_time; \
- ld r12,_MSR(r1); \
- andi. r10,r12,MSR_PR; /* Restore cr0 (coming from user) */ \
-33: \
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
-
-#else /* CONFIG_PPC_SPLPAR */
-#define ACCOUNT_STOLEN_TIME
-
-#endif /* CONFIG_PPC_SPLPAR */
-
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
/*
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index dc728bb1c89a..e6b5a362fc91 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -398,7 +398,6 @@ exc_##n##_common: \
std r10,_NIP(r1); /* save SRR0 to stackframe */ \
std r11,_MSR(r1); /* save SRR1 to stackframe */ \
beq 2f; /* if from kernel mode */ \
- ACCOUNT_CPU_USER_ENTRY(r13,r10,r11);/* accounting (uses cr0+eq) */ \
2: ld r3,excf+EX_R10(r13); /* get back r10 */ \
ld r4,excf+EX_R11(r13); /* get back r11 */ \
mfspr r5,scratch; /* get back r13 */ \
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index b36247ad1f64..121a55c87c02 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -577,7 +577,6 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real)
kuap_save_amr_and_lock r9, r10, cr1, cr0
.endif
beq 101f /* if from kernel mode */
- ACCOUNT_CPU_USER_ENTRY(r13, r9, r10)
BEGIN_FTR_SECTION
ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */
std r9,_PPR(r1)
@@ -645,10 +644,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
ld r11,exception_marker@toc(r2)
std r10,RESULT(r1) /* clear regs->result */
std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */
-
- .if ISTACK
- ACCOUNT_STOLEN_TIME
- .endif
.endm
/*
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 14/18] powerpc: move NMI entry/exit code into wrapper
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (12 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 13/18] powerpc/64: entry cpu time accounting in C Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 15/18] powerpc/64s: move NMI soft-mask handling to C Nicholas Piggin
` (3 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
This moves the common NMI entry and exit code into the interrupt handler
wrappers.
This changes the behaviour of soft-NMI (watchdog) and HMI interrupts, and
also MCE interrupts on 64e, by adding missing parts of the NMI entry to
them. It fixes a bug with sreset on pseries HPT guests which shouldn't
call nmi_enter().
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 24 ++++++++++++++++++
arch/powerpc/kernel/mce.c | 11 --------
arch/powerpc/kernel/traps.c | 38 +++++-----------------------
arch/powerpc/kernel/watchdog.c | 10 +++-----
4 files changed, 35 insertions(+), 48 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 9bd14c59dc8e..a87284707f09 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -69,14 +69,38 @@ static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct int
}
struct interrupt_nmi_state {
+#ifdef CONFIG_PPC64
+ u8 ftrace_enabled;
+#endif
};
static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
{
+#ifdef CONFIG_PPC64
+ state->ftrace_enabled = this_cpu_get_ftrace_enabled();
+ this_cpu_set_ftrace_enabled(0);
+#endif
+
+ /*
+ * Do not use nmi_enter() for pseries hash guest taking a real-mode
+ * NMI because not everything it touches is within the RMA limit.
+ */
+ if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) ||
+ !firmware_has_feature(FW_FEATURE_LPAR) ||
+ radix_enabled() || (mfmsr() & MSR_DR))
+ nmi_enter();
}
static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
{
+ if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) ||
+ !firmware_has_feature(FW_FEATURE_LPAR) ||
+ radix_enabled() || (mfmsr() & MSR_DR))
+ nmi_exit();
+
+#ifdef CONFIG_PPC64
+ this_cpu_set_ftrace_enabled(state->ftrace_enabled);
+#endif
}
/**
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
index b84459f45b1a..9f39deed4fca 100644
--- a/arch/powerpc/kernel/mce.c
+++ b/arch/powerpc/kernel/mce.c
@@ -592,12 +592,6 @@ EXPORT_SYMBOL_GPL(machine_check_print_event_info);
DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early)
{
long handled = 0;
- u8 ftrace_enabled = this_cpu_get_ftrace_enabled();
-
- this_cpu_set_ftrace_enabled(0);
- /* Do not use nmi_enter/exit for pseries hpte guest */
- if (radix_enabled() || !firmware_has_feature(FW_FEATURE_LPAR))
- nmi_enter();
hv_nmi_check_nonrecoverable(regs);
@@ -607,11 +601,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early)
if (ppc_md.machine_check_early)
handled = ppc_md.machine_check_early(regs);
- if (radix_enabled() || !firmware_has_feature(FW_FEATURE_LPAR))
- nmi_exit();
-
- this_cpu_set_ftrace_enabled(ftrace_enabled);
-
return handled;
}
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 09780481e1b1..95f84c542523 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -443,11 +443,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception)
{
unsigned long hsrr0, hsrr1;
bool saved_hsrrs = false;
- u8 ftrace_enabled = this_cpu_get_ftrace_enabled();
-
- this_cpu_set_ftrace_enabled(0);
-
- nmi_enter();
/*
* System reset can interrupt code where HSRRs are live and MSR[RI]=1.
@@ -519,10 +514,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception)
mtspr(SPRN_HSRR1, hsrr1);
}
- nmi_exit();
-
- this_cpu_set_ftrace_enabled(ftrace_enabled);
-
/* What should we do here? We could issue a shutdown or hard reset. */
return 0;
@@ -824,6 +815,12 @@ int machine_check_generic(struct pt_regs *regs)
#endif /* everything else */
+/*
+ * BOOK3S_64 does not call this handler as a non-maskable interrupt
+ * (it uses its own early real-mode handler to handle the MCE proper
+ * and then raises irq_work to call this handler when interrupts are
+ * enabled).
+ */
#ifdef CONFIG_PPC_BOOK3S_64
DEFINE_INTERRUPT_HANDLER_ASYNC(machine_check_exception)
#else
@@ -832,20 +829,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(machine_check_exception)
{
int recover = 0;
- /*
- * BOOK3S_64 does not call this handler as a non-maskable interrupt
- * (it uses its own early real-mode handler to handle the MCE proper
- * and then raises irq_work to call this handler when interrupts are
- * enabled).
- *
- * This is silly. The BOOK3S_64 should just call a different function
- * rather than expecting semantics to magically change. Something
- * like 'non_nmi_machine_check_exception()', perhaps?
- */
- const bool nmi = !IS_ENABLED(CONFIG_PPC_BOOK3S_64);
-
- if (nmi) nmi_enter();
-
__this_cpu_inc(irq_stat.mce_exceptions);
add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
@@ -870,24 +853,17 @@ DEFINE_INTERRUPT_HANDLER_NMI(machine_check_exception)
if (check_io_access(regs))
goto bail;
- if (nmi) nmi_exit();
-
die("Machine check", regs, SIGBUS);
/* Must die if the interrupt is not recoverable */
if (!(regs->msr & MSR_RI))
die("Unrecoverable Machine check", regs, SIGBUS);
-#ifdef CONFIG_PPC_BOOK3S_64
bail:
+#ifdef CONFIG_PPC_BOOK3S_64
return;
#else
return 0;
-
-bail:
- if (nmi) nmi_exit();
-
- return 0;
#endif
}
diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c
index 824b9376ac35..dc39534836a3 100644
--- a/arch/powerpc/kernel/watchdog.c
+++ b/arch/powerpc/kernel/watchdog.c
@@ -254,11 +254,12 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)
int cpu = raw_smp_processor_id();
u64 tb;
+ /* should only arrive from kernel, with irqs disabled */
+ WARN_ON_ONCE(!arch_irq_disabled_regs(regs));
+
if (!cpumask_test_cpu(cpu, &wd_cpus_enabled))
return 0;
- nmi_enter();
-
__this_cpu_inc(irq_stat.soft_nmi_irqs);
tb = get_tb();
@@ -266,7 +267,7 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)
wd_smp_lock(&flags);
if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) {
wd_smp_unlock(&flags);
- goto out;
+ return 0;
}
set_cpu_stuck(cpu, tb);
@@ -290,9 +291,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)
if (wd_panic_timeout_tb < 0x7fffffff)
mtspr(SPRN_DEC, wd_panic_timeout_tb);
-out:
- nmi_exit();
-
return 0;
}
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 15/18] powerpc/64s: move NMI soft-mask handling to C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (13 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 14/18] powerpc: move NMI entry/exit code into wrapper Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 16/18] powerpc/64s: runlatch interrupt handling in C Nicholas Piggin
` (2 subsequent siblings)
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Saving and restoring soft-mask state can now be done in C using the
interrupt handler wrapper functions.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 25 ++++++++++++
arch/powerpc/kernel/exceptions-64s.S | 60 ----------------------------
2 files changed, 25 insertions(+), 60 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index a87284707f09..5ece57dd0455 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -70,6 +70,10 @@ static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct int
struct interrupt_nmi_state {
#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3S_64
+ u8 irq_soft_mask;
+ u8 irq_happened;
+#endif
u8 ftrace_enabled;
#endif
};
@@ -77,6 +81,20 @@ struct interrupt_nmi_state {
static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
{
#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3S_64
+ state->irq_soft_mask = local_paca->irq_soft_mask;
+ state->irq_happened = local_paca->irq_happened;
+ state->ftrace_enabled = this_cpu_get_ftrace_enabled();
+
+ /*
+ * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
+ * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile
+ * because that goes through irq tracing which we don't want in NMI.
+ */
+ local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
+ local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
+#endif
+
state->ftrace_enabled = this_cpu_get_ftrace_enabled();
this_cpu_set_ftrace_enabled(0);
#endif
@@ -100,6 +118,13 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
#ifdef CONFIG_PPC64
this_cpu_set_ftrace_enabled(state->ftrace_enabled);
+
+#ifdef CONFIG_PPC_BOOK3S_64
+ /* Check we didn't change the pending interrupt mask. */
+ WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened);
+ local_paca->irq_happened = state->irq_happened;
+ local_paca->irq_soft_mask = state->irq_soft_mask;
+#endif
#endif
}
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 121a55c87c02..0949dd47be59 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1010,20 +1010,6 @@ EXC_COMMON_BEGIN(system_reset_common)
ld r1,PACA_NMI_EMERG_SP(r13)
subi r1,r1,INT_FRAME_SIZE
__GEN_COMMON_BODY system_reset
- /*
- * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
- * the right thing. We do not want to reconcile because that goes
- * through irq tracing which we don't want in NMI.
- *
- * Save PACAIRQHAPPENED to RESULT (otherwise unused), and set HARD_DIS
- * as we are running with MSR[EE]=0.
- */
- li r10,IRQS_ALL_DISABLED
- stb r10,PACAIRQSOFTMASK(r13)
- lbz r10,PACAIRQHAPPENED(r13)
- std r10,RESULT(r1)
- ori r10,r10,PACA_IRQ_HARD_DIS
- stb r10,PACAIRQHAPPENED(r13)
addi r3,r1,STACK_FRAME_OVERHEAD
bl system_reset_exception
@@ -1039,14 +1025,6 @@ EXC_COMMON_BEGIN(system_reset_common)
subi r10,r10,1
sth r10,PACA_IN_NMI(r13)
- /*
- * Restore soft mask settings.
- */
- ld r10,RESULT(r1)
- stb r10,PACAIRQHAPPENED(r13)
- ld r10,SOFTE(r1)
- stb r10,PACAIRQSOFTMASK(r13)
-
kuap_restore_amr r9, r10
EXCEPTION_RESTORE_REGS
RFI_TO_USER_OR_KERNEL
@@ -1192,30 +1170,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
li r10,MSR_RI
mtmsrd r10,1
- /*
- * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see
- * system_reset_common)
- */
- li r10,IRQS_ALL_DISABLED
- stb r10,PACAIRQSOFTMASK(r13)
- lbz r10,PACAIRQHAPPENED(r13)
- std r10,RESULT(r1)
- ori r10,r10,PACA_IRQ_HARD_DIS
- stb r10,PACAIRQHAPPENED(r13)
-
addi r3,r1,STACK_FRAME_OVERHEAD
bl machine_check_early
std r3,RESULT(r1) /* Save result */
ld r12,_MSR(r1)
- /*
- * Restore soft mask settings.
- */
- ld r10,RESULT(r1)
- stb r10,PACAIRQHAPPENED(r13)
- ld r10,SOFTE(r1)
- stb r10,PACAIRQSOFTMASK(r13)
-
#ifdef CONFIG_PPC_P7_NAP
/*
* Check if thread was in power saving mode. We come here when any
@@ -2814,17 +2773,6 @@ EXC_COMMON_BEGIN(soft_nmi_common)
subi r1,r1,INT_FRAME_SIZE
__GEN_COMMON_BODY soft_nmi
- /*
- * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see
- * system_reset_common)
- */
- li r10,IRQS_ALL_DISABLED
- stb r10,PACAIRQSOFTMASK(r13)
- lbz r10,PACAIRQHAPPENED(r13)
- std r10,RESULT(r1)
- ori r10,r10,PACA_IRQ_HARD_DIS
- stb r10,PACAIRQHAPPENED(r13)
-
addi r3,r1,STACK_FRAME_OVERHEAD
bl soft_nmi_interrupt
@@ -2832,14 +2780,6 @@ EXC_COMMON_BEGIN(soft_nmi_common)
li r9,0
mtmsrd r9,1
- /*
- * Restore soft mask settings.
- */
- ld r10,RESULT(r1)
- stb r10,PACAIRQHAPPENED(r13)
- ld r10,SOFTE(r1)
- stb r10,PACAIRQSOFTMASK(r13)
-
kuap_restore_amr r9, r10
EXCEPTION_RESTORE_REGS hsrr=0
RFI_TO_KERNEL
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 16/18] powerpc/64s: runlatch interrupt handling in C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (14 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 15/18] powerpc/64s: move NMI soft-mask handling to C Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 17/18] powerpc/64s: power4 nap fixup " Nicholas Piggin
2020-11-05 14:34 ` [PATCH 18/18] powerpc/64s: move power4 idle entirely to C Nicholas Piggin
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
There is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 7 +++++++
arch/powerpc/kernel/exceptions-64s.S | 18 ------------------
2 files changed, 7 insertions(+), 18 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 5ece57dd0455..46846fe332b9 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -6,6 +6,7 @@
#include <linux/hardirq.h>
#include <asm/cputime.h>
#include <asm/ftrace.h>
+#include <asm/runlatch.h>
struct interrupt_state {
#ifdef CONFIG_PPC_BOOK3E_64
@@ -60,6 +61,12 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt
static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (cpu_has_feature(CPU_FTR_CTRL) &&
+ !test_thread_local_flags(_TLF_RUNLATCH))
+ __ppc64_runlatch_on();
+#endif
+
interrupt_enter_prepare(regs, state);
}
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 0949dd47be59..227bad3a586d 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -692,14 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
ld r1,GPR1(r1)
.endm
-#define RUNLATCH_ON \
-BEGIN_FTR_SECTION \
- ld r3, PACA_THREAD_INFO(r13); \
- ld r4,TI_LOCAL_FLAGS(r3); \
- andi. r0,r4,_TLF_RUNLATCH; \
- beql ppc64_runlatch_on_trampoline; \
-END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
-
/*
* When the idle code in power4_idle puts the CPU into NAP mode,
* it has to do so in a loop, and relies on the external interrupt
@@ -1581,7 +1573,6 @@ EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
EXC_COMMON_BEGIN(hardware_interrupt_common)
GEN_COMMON hardware_interrupt
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_IRQ
b interrupt_return
@@ -1767,7 +1758,6 @@ EXC_VIRT_END(decrementer, 0x4900, 0x80)
EXC_COMMON_BEGIN(decrementer_common)
GEN_COMMON decrementer
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
bl timer_interrupt
b interrupt_return
@@ -1853,7 +1843,6 @@ EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
EXC_COMMON_BEGIN(doorbell_super_common)
GEN_COMMON doorbell_super
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_DOORBELL
bl doorbell_exception
@@ -2208,7 +2197,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common)
EXC_COMMON_BEGIN(hmi_exception_common)
GEN_COMMON hmi_exception
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
bl handle_hmi_exception
b interrupt_return
@@ -2238,7 +2226,6 @@ EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
EXC_COMMON_BEGIN(h_doorbell_common)
GEN_COMMON h_doorbell
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_DOORBELL
bl doorbell_exception
@@ -2272,7 +2259,6 @@ EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
EXC_COMMON_BEGIN(h_virt_irq_common)
GEN_COMMON h_virt_irq
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_IRQ
b interrupt_return
@@ -2319,7 +2305,6 @@ EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
EXC_COMMON_BEGIN(performance_monitor_common)
GEN_COMMON performance_monitor
FINISH_NAP
- RUNLATCH_ON
addi r3,r1,STACK_FRAME_OVERHEAD
bl performance_monitor_exception
b interrupt_return
@@ -3030,9 +3015,6 @@ kvmppc_skip_Hinterrupt:
* come here.
*/
-EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
- b __ppc64_runlatch_on
-
USE_FIXED_SECTION(virt_trampolines)
/*
* All code below __end_interrupts is treated as soft-masked. If
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 17/18] powerpc/64s: power4 nap fixup in C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (15 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 16/18] powerpc/64s: runlatch interrupt handling in C Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-05 14:34 ` [PATCH 18/18] powerpc/64s: move power4 idle entirely to C Nicholas Piggin
17 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
There is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 15 +++++++++
arch/powerpc/include/asm/processor.h | 1 +
arch/powerpc/include/asm/thread_info.h | 6 ++++
arch/powerpc/kernel/exceptions-64s.S | 45 --------------------------
arch/powerpc/kernel/idle_book3s.S | 4 +++
5 files changed, 26 insertions(+), 45 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 46846fe332b9..133dbe9db12b 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -8,6 +8,16 @@
#include <asm/ftrace.h>
#include <asm/runlatch.h>
+static inline void nap_adjust_return(struct pt_regs *regs)
+{
+#ifdef CONFIG_PPC_970_NAP
+ if (unlikely(test_thread_local_flags(_TLF_NAPPING))) {
+ clear_thread_local_flags(_TLF_NAPPING);
+ regs->nip = (unsigned long)power4_idle_nap_return;
+ }
+#endif
+}
+
struct interrupt_state {
#ifdef CONFIG_PPC_BOOK3E_64
enum ctx_state ctx_state;
@@ -73,6 +83,9 @@ static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct in
static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state)
{
interrupt_exit_prepare(regs, state);
+
+ /* Adjust at exit so the main handler sees the true NIA */
+ nap_adjust_return(regs);
}
struct interrupt_nmi_state {
@@ -123,6 +136,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
radix_enabled() || (mfmsr() & MSR_DR))
nmi_exit();
+ nap_adjust_return(regs);
+
#ifdef CONFIG_PPC64
this_cpu_set_ftrace_enabled(state->ftrace_enabled);
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index c61c859b51a8..d845850f75e2 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -418,6 +418,7 @@ extern unsigned long isa300_idle_stop_mayloss(unsigned long psscr_val);
extern unsigned long isa206_idle_insn_mayloss(unsigned long type);
#ifdef CONFIG_PPC_970_NAP
extern void power4_idle_nap(void);
+void power4_idle_nap_return(void);
#endif
extern unsigned long cpuidle_disable;
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index c9443c16e5fb..e7ee220b88ea 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -151,6 +151,12 @@ void arch_setup_new_exec(void);
#ifndef __ASSEMBLY__
+static inline void clear_thread_local_flags(unsigned int flags)
+{
+ struct thread_info *ti = current_thread_info();
+ ti->local_flags &= ~flags;
+}
+
static inline bool test_thread_local_flags(unsigned int flags)
{
struct thread_info *ti = current_thread_info();
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 227bad3a586d..1db6b3438c88 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -692,25 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
ld r1,GPR1(r1)
.endm
-/*
- * When the idle code in power4_idle puts the CPU into NAP mode,
- * it has to do so in a loop, and relies on the external interrupt
- * and decrementer interrupt entry code to get it out of the loop.
- * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
- * to signal that it is in the loop and needs help to get out.
- */
-#ifdef CONFIG_PPC_970_NAP
-#define FINISH_NAP \
-BEGIN_FTR_SECTION \
- ld r11, PACA_THREAD_INFO(r13); \
- ld r9,TI_LOCAL_FLAGS(r11); \
- andi. r10,r9,_TLF_NAPPING; \
- bnel power4_fixup_nap; \
-END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
-#else
-#define FINISH_NAP
-#endif
-
/*
* There are a few constraints to be concerned with.
* - Real mode exceptions code/data must be located at their physical location.
@@ -1250,7 +1231,6 @@ EXC_COMMON_BEGIN(machine_check_common)
*/
GEN_COMMON machine_check
- FINISH_NAP
/* Enable MSR_RI when finished with PACA_EXMC */
li r10,MSR_RI
mtmsrd r10,1
@@ -1572,7 +1552,6 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
EXC_COMMON_BEGIN(hardware_interrupt_common)
GEN_COMMON hardware_interrupt
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_IRQ
b interrupt_return
@@ -1757,7 +1736,6 @@ EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
EXC_VIRT_END(decrementer, 0x4900, 0x80)
EXC_COMMON_BEGIN(decrementer_common)
GEN_COMMON decrementer
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
bl timer_interrupt
b interrupt_return
@@ -1842,7 +1820,6 @@ EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100)
EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
EXC_COMMON_BEGIN(doorbell_super_common)
GEN_COMMON doorbell_super
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_DOORBELL
bl doorbell_exception
@@ -2196,7 +2173,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common)
EXC_COMMON_BEGIN(hmi_exception_common)
GEN_COMMON hmi_exception
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
bl handle_hmi_exception
b interrupt_return
@@ -2225,7 +2201,6 @@ EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20)
EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
EXC_COMMON_BEGIN(h_doorbell_common)
GEN_COMMON h_doorbell
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_DOORBELL
bl doorbell_exception
@@ -2258,7 +2233,6 @@ EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20)
EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
EXC_COMMON_BEGIN(h_virt_irq_common)
GEN_COMMON h_virt_irq
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_IRQ
b interrupt_return
@@ -2304,7 +2278,6 @@ EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20)
EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
EXC_COMMON_BEGIN(performance_monitor_common)
GEN_COMMON performance_monitor
- FINISH_NAP
addi r3,r1,STACK_FRAME_OVERHEAD
bl performance_monitor_exception
b interrupt_return
@@ -3032,24 +3005,6 @@ USE_FIXED_SECTION(virt_trampolines)
__end_interrupts:
DEFINE_FIXED_SYMBOL(__end_interrupts)
-#ifdef CONFIG_PPC_970_NAP
- /*
- * Called by exception entry code if _TLF_NAPPING was set, this clears
- * the NAPPING flag, and redirects the exception exit to
- * power4_fixup_nap_return.
- */
- .globl power4_fixup_nap
-EXC_COMMON_BEGIN(power4_fixup_nap)
- andc r9,r9,r10
- std r9,TI_LOCAL_FLAGS(r11)
- LOAD_REG_ADDR(r10, power4_idle_nap_return)
- std r10,_NIP(r1)
- blr
-
-power4_idle_nap_return:
- blr
-#endif
-
CLOSE_FIXED_SECTION(real_vectors);
CLOSE_FIXED_SECTION(real_trampolines);
CLOSE_FIXED_SECTION(virt_vectors);
diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S
index 22f249b6f58d..27d2e6a72ec9 100644
--- a/arch/powerpc/kernel/idle_book3s.S
+++ b/arch/powerpc/kernel/idle_book3s.S
@@ -201,4 +201,8 @@ _GLOBAL(power4_idle_nap)
mtmsrd r7
isync
b 1b
+
+ .globl power4_idle_nap_return
+power4_idle_nap_return:
+ blr
#endif
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH 18/18] powerpc/64s: move power4 idle entirely to C
2020-11-05 14:34 [PATCH 00/18] powerpc: interrupt wrappers Nicholas Piggin
` (16 preceding siblings ...)
2020-11-05 14:34 ` [PATCH 17/18] powerpc/64s: power4 nap fixup " Nicholas Piggin
@ 2020-11-05 14:34 ` Nicholas Piggin
2020-11-06 7:34 ` kernel test robot
2020-11-07 9:43 ` Christophe Leroy
17 siblings, 2 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-05 14:34 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Christophe asked about doing this, most of the code is still in
asm but maybe it's slightly nicer? I don't know if it's worthwhile.
---
arch/powerpc/kernel/idle.c | 25 ++++++++++++++++++++-----
arch/powerpc/kernel/idle_book3s.S | 22 ----------------------
2 files changed, 20 insertions(+), 27 deletions(-)
diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index ae0e2632393d..849e77a45915 100644
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -72,6 +72,9 @@ int powersave_nap;
#ifdef CONFIG_PPC_970_NAP
void power4_idle(void)
{
+ unsigned long msr_idle = MSR_KERNEL|MSR_EE|MSR_POW;
+ unsigned long tmp1, tmp2;
+
if (!cpu_has_feature(CPU_FTR_CAN_NAP))
return;
@@ -84,13 +87,25 @@ void power4_idle(void)
if (cpu_has_feature(CPU_FTR_ALTIVEC))
asm volatile("DSSALL ; sync" ::: "memory");
- power4_idle_nap();
-
+ asm volatile(
+" ld %0,PACA_THREAD_INFO(r13) \n"
+" ld %1,TI_LOCAL_FLAGS(%0) \n"
+" ori %1,%1,_TLF_NAPPING \n"
+" std %1,TI_LOCAL_FLAGS(%0) \n"
/*
- * power4_idle_nap returns with interrupts enabled (soft and hard).
- * to our caller with interrupts enabled (soft and hard). Our caller
- * can cope with either interrupts disabled or enabled upon return.
+ * NAPPING bit is set, from this point onward nap_adjust_return()
+ * will cause interrupts to return to power4_idle_nap_return.
*/
+"1: sync \n"
+" isync \n"
+" mtmsrd %2 \n"
+" isync \n"
+" b 1b \n"
+" .globl power4_idle_nap_return \n"
+"power4_idle_nap_return: \n"
+ : "=r"(tmp1), "=r"(tmp2)
+ : "r"(msr_idle)
+ );
}
#endif
diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S
index 27d2e6a72ec9..d4047f3c672e 100644
--- a/arch/powerpc/kernel/idle_book3s.S
+++ b/arch/powerpc/kernel/idle_book3s.S
@@ -184,25 +184,3 @@ _GLOBAL(isa206_idle_insn_mayloss)
IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
2: IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
#endif
-
-#ifdef CONFIG_PPC_970_NAP
-_GLOBAL(power4_idle_nap)
- LOAD_REG_IMMEDIATE(r7, MSR_KERNEL|MSR_EE|MSR_POW)
- ld r9,PACA_THREAD_INFO(r13)
- ld r8,TI_LOCAL_FLAGS(r9)
- ori r8,r8,_TLF_NAPPING
- std r8,TI_LOCAL_FLAGS(r9)
- /*
- * NAPPING bit is set, from this point onward power4_fixup_nap
- * will cause exceptions to return to power4_idle_nap_return.
- */
-1: sync
- isync
- mtmsrd r7
- isync
- b 1b
-
- .globl power4_idle_nap_return
-power4_idle_nap_return:
- blr
-#endif
--
2.23.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH 18/18] powerpc/64s: move power4 idle entirely to C
2020-11-05 14:34 ` [PATCH 18/18] powerpc/64s: move power4 idle entirely to C Nicholas Piggin
@ 2020-11-06 7:34 ` kernel test robot
2020-11-07 9:43 ` Christophe Leroy
1 sibling, 0 replies; 36+ messages in thread
From: kernel test robot @ 2020-11-06 7:34 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev; +Cc: kbuild-all, Nicholas Piggin
[-- Attachment #1: Type: text/plain, Size: 1626 bytes --]
Hi Nicholas,
I love your patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on v5.10-rc2 next-20201105]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-allyesconfig (attached as .config)
compiler: powerpc64-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/561aae13dd3400b772b91d60681d55937e046cbc
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Nicholas-Piggin/powerpc-interrupt-wrappers/20201105-231909
git checkout 561aae13dd3400b772b91d60681d55937e046cbc
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
{standard input}: Assembler messages:
>> {standard input}:662: Error: unsupported relocation against r13
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 71461 bytes --]
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 18/18] powerpc/64s: move power4 idle entirely to C
2020-11-05 14:34 ` [PATCH 18/18] powerpc/64s: move power4 idle entirely to C Nicholas Piggin
2020-11-06 7:34 ` kernel test robot
@ 2020-11-07 9:43 ` Christophe Leroy
2020-11-10 8:43 ` Nicholas Piggin
1 sibling, 1 reply; 36+ messages in thread
From: Christophe Leroy @ 2020-11-07 9:43 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev
Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
> Christophe asked about doing this, most of the code is still in
> asm but maybe it's slightly nicer? I don't know if it's worthwhile.
Heu... I don't think I was asking for that, but why not, see later comments.
At first I was just asking to write the following in C:
+
+ .globl power4_idle_nap_return
+power4_idle_nap_return:
+ blr
In extenso, instead of the above do somewhere something like:
void power4_idle_nap_return(void)
{
}
> ---
> arch/powerpc/kernel/idle.c | 25 ++++++++++++++++++++-----
> arch/powerpc/kernel/idle_book3s.S | 22 ----------------------
> 2 files changed, 20 insertions(+), 27 deletions(-)
>
> diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
> index ae0e2632393d..849e77a45915 100644
> --- a/arch/powerpc/kernel/idle.c
> +++ b/arch/powerpc/kernel/idle.c
> @@ -72,6 +72,9 @@ int powersave_nap;
> #ifdef CONFIG_PPC_970_NAP
> void power4_idle(void)
> {
> + unsigned long msr_idle = MSR_KERNEL|MSR_EE|MSR_POW;
> + unsigned long tmp1, tmp2;
> +
> if (!cpu_has_feature(CPU_FTR_CAN_NAP))
> return;
>
> @@ -84,13 +87,25 @@ void power4_idle(void)
> if (cpu_has_feature(CPU_FTR_ALTIVEC))
> asm volatile("DSSALL ; sync" ::: "memory");
>
> - power4_idle_nap();
> -
> + asm volatile(
> +" ld %0,PACA_THREAD_INFO(r13) \n"
> +" ld %1,TI_LOCAL_FLAGS(%0) \n"
> +" ori %1,%1,_TLF_NAPPING \n"
> +" std %1,TI_LOCAL_FLAGS(%0) \n"
Can't this just be:
current_thread_info()->local_flags |= _TLF_NAPPING;
> /*
> - * power4_idle_nap returns with interrupts enabled (soft and hard).
> - * to our caller with interrupts enabled (soft and hard). Our caller
> - * can cope with either interrupts disabled or enabled upon return.
> + * NAPPING bit is set, from this point onward nap_adjust_return()
> + * will cause interrupts to return to power4_idle_nap_return.
> */
> +"1: sync \n"
> +" isync \n"
> +" mtmsrd %2 \n"
> +" isync \n"
> +" b 1b \n"
And this:
for (;;) {
mb();
isync();
mtmsr(MSR_KERNEL|MSR_EE|MSR_POW);
isync();
}
> +" .globl power4_idle_nap_return \n"
> +"power4_idle_nap_return: \n"
> + : "=r"(tmp1), "=r"(tmp2)
> + : "r"(msr_idle)
> + );
> }
> #endif
>
Christophe
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH 18/18] powerpc/64s: move power4 idle entirely to C
2020-11-07 9:43 ` Christophe Leroy
@ 2020-11-10 8:43 ` Nicholas Piggin
0 siblings, 0 replies; 36+ messages in thread
From: Nicholas Piggin @ 2020-11-10 8:43 UTC (permalink / raw)
To: Christophe Leroy, linuxppc-dev
Excerpts from Christophe Leroy's message of November 7, 2020 7:43 pm:
>
>
> Le 05/11/2020 à 15:34, Nicholas Piggin a écrit :
>> Christophe asked about doing this, most of the code is still in
>> asm but maybe it's slightly nicer? I don't know if it's worthwhile.
>
> Heu... I don't think I was asking for that, but why not, see later comments.
>
> At first I was just asking to write the following in C:
>
> +
> + .globl power4_idle_nap_return
> +power4_idle_nap_return:
> + blr
>
>
> In extenso, instead of the above do somewhere something like:
>
> void power4_idle_nap_return(void)
> {
> }
Ah! Well either was a good question. I don't mind attempting it :)
>> ---
>> arch/powerpc/kernel/idle.c | 25 ++++++++++++++++++++-----
>> arch/powerpc/kernel/idle_book3s.S | 22 ----------------------
>> 2 files changed, 20 insertions(+), 27 deletions(-)
>>
>> diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
>> index ae0e2632393d..849e77a45915 100644
>> --- a/arch/powerpc/kernel/idle.c
>> +++ b/arch/powerpc/kernel/idle.c
>> @@ -72,6 +72,9 @@ int powersave_nap;
>> #ifdef CONFIG_PPC_970_NAP
>> void power4_idle(void)
>> {
>> + unsigned long msr_idle = MSR_KERNEL|MSR_EE|MSR_POW;
>> + unsigned long tmp1, tmp2;
>> +
>> if (!cpu_has_feature(CPU_FTR_CAN_NAP))
>> return;
>>
>> @@ -84,13 +87,25 @@ void power4_idle(void)
>> if (cpu_has_feature(CPU_FTR_ALTIVEC))
>> asm volatile("DSSALL ; sync" ::: "memory");
>>
>> - power4_idle_nap();
>> -
>> + asm volatile(
>> +" ld %0,PACA_THREAD_INFO(r13) \n"
>> +" ld %1,TI_LOCAL_FLAGS(%0) \n"
>> +" ori %1,%1,_TLF_NAPPING \n"
>> +" std %1,TI_LOCAL_FLAGS(%0) \n"
>
> Can't this just be:
>
> current_thread_info()->local_flags |= _TLF_NAPPING;
>
>> /*
>> - * power4_idle_nap returns with interrupts enabled (soft and hard).
>> - * to our caller with interrupts enabled (soft and hard). Our caller
>> - * can cope with either interrupts disabled or enabled upon return.
>> + * NAPPING bit is set, from this point onward nap_adjust_return()
>> + * will cause interrupts to return to power4_idle_nap_return.
>> */
>> +"1: sync \n"
>> +" isync \n"
>> +" mtmsrd %2 \n"
>> +" isync \n"
>> +" b 1b \n"
>
> And this:
>
> for (;;) {
> mb();
> isync();
> mtmsr(MSR_KERNEL|MSR_EE|MSR_POW);
> isync();
> }
I was hoping something nicer like this but I think not because as soon
as we set _TLF_NAPPING, we might take an interrupt which returns
somewhere else, and you aren't allowed to do that in C code (mainly
because the stack and register state would be unknown). Even going
immediately to blr or end of function might miss restoring NVGPRs etc.
There might be some tricks we could play with soft-masking interrupts,
using MSR[EE]=0, and then doing all this and returning to right after
the mtmsr POW with a flag set... But it's a bit of tricky churn for an
old CPU that works okay.
Thanks,
Nick
>
>
>> +" .globl power4_idle_nap_return \n"
>> +"power4_idle_nap_return: \n"
>> + : "=r"(tmp1), "=r"(tmp2)
>> + : "r"(msr_idle)
>> + );
>> }
>> #endif
>>
>
> Christophe
>
^ permalink raw reply [flat|nested] 36+ messages in thread