From: Andy Lutomirski <luto@kernel.org>
To: x86@kernel.org
Cc: LKML <linux-kernel@vger.kernel.org>,
Dave Hansen <dave.hansen@linux.intel.com>,
Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Yonghong Song <yhs@fb.com>,
Masami Hiramatsu <mhiramat@kernel.org>,
Andy Lutomirski <luto@kernel.org>,
Peter Zijlstra <peterz@infradead.org>
Subject: [PATCH 02/11] x86/fault: Fold mm_fault_error() into do_user_addr_fault()
Date: Sun, 31 Jan 2021 09:24:33 -0800 [thread overview]
Message-ID: <c5adc9322511167403edd334ca760466535f582d.1612113550.git.luto@kernel.org> (raw)
In-Reply-To: <cover.1612113550.git.luto@kernel.org>
mm_fault_error() is logically just the end of do_user_addr_fault().
Combine the functions. This makes the code easier to read.
Most of the churn here is from renaming hw_error_code to error_code in
do_user_addr_fault().
This makes no difference at all to the generated code (objdump -dr) as
compared to changing noinline to __always_inline in the definition of
mm_fault_error().
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
arch/x86/mm/fault.c | 97 +++++++++++++++++++++------------------------
1 file changed, 45 insertions(+), 52 deletions(-)
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 50dfdc71761e..aff35c9ba018 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -944,40 +944,6 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
}
-static noinline void
-mm_fault_error(struct pt_regs *regs, unsigned long error_code,
- unsigned long address, vm_fault_t fault)
-{
- if (fatal_signal_pending(current) && !(error_code & X86_PF_USER)) {
- no_context(regs, error_code, address, 0, 0);
- return;
- }
-
- if (fault & VM_FAULT_OOM) {
- /* Kernel mode? Handle exceptions or die: */
- if (!(error_code & X86_PF_USER)) {
- no_context(regs, error_code, address,
- SIGSEGV, SEGV_MAPERR);
- return;
- }
-
- /*
- * We ran out of memory, call the OOM killer, and return the
- * userspace (which will retry the fault, or kill us if we got
- * oom-killed):
- */
- pagefault_out_of_memory();
- } else {
- if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
- VM_FAULT_HWPOISON_LARGE))
- do_sigbus(regs, error_code, address, fault);
- else if (fault & VM_FAULT_SIGSEGV)
- bad_area_nosemaphore(regs, error_code, address);
- else
- BUG();
- }
-}
-
static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)
{
if ((error_code & X86_PF_WRITE) && !pte_write(*pte))
@@ -1215,7 +1181,7 @@ NOKPROBE_SYMBOL(do_kern_addr_fault);
/* Handle faults in the user portion of the address space */
static inline
void do_user_addr_fault(struct pt_regs *regs,
- unsigned long hw_error_code,
+ unsigned long error_code,
unsigned long address)
{
struct vm_area_struct *vma;
@@ -1235,8 +1201,8 @@ void do_user_addr_fault(struct pt_regs *regs,
* Reserved bits are never expected to be set on
* entries in the user portion of the page tables.
*/
- if (unlikely(hw_error_code & X86_PF_RSVD))
- pgtable_bad(regs, hw_error_code, address);
+ if (unlikely(error_code & X86_PF_RSVD))
+ pgtable_bad(regs, error_code, address);
/*
* If SMAP is on, check for invalid kernel (supervisor) access to user
@@ -1246,10 +1212,10 @@ void do_user_addr_fault(struct pt_regs *regs,
* enforcement appears to be consistent with the USER bit.
*/
if (unlikely(cpu_feature_enabled(X86_FEATURE_SMAP) &&
- !(hw_error_code & X86_PF_USER) &&
+ !(error_code & X86_PF_USER) &&
!(regs->flags & X86_EFLAGS_AC)))
{
- bad_area_nosemaphore(regs, hw_error_code, address);
+ bad_area_nosemaphore(regs, error_code, address);
return;
}
@@ -1258,7 +1224,7 @@ void do_user_addr_fault(struct pt_regs *regs,
* in a region with pagefaults disabled then we must not take the fault
*/
if (unlikely(faulthandler_disabled() || !mm)) {
- bad_area_nosemaphore(regs, hw_error_code, address);
+ bad_area_nosemaphore(regs, error_code, address);
return;
}
@@ -1279,9 +1245,9 @@ void do_user_addr_fault(struct pt_regs *regs,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
- if (hw_error_code & X86_PF_WRITE)
+ if (error_code & X86_PF_WRITE)
flags |= FAULT_FLAG_WRITE;
- if (hw_error_code & X86_PF_INSTR)
+ if (error_code & X86_PF_INSTR)
flags |= FAULT_FLAG_INSTRUCTION;
#ifdef CONFIG_X86_64
@@ -1297,7 +1263,7 @@ void do_user_addr_fault(struct pt_regs *regs,
* to consider the PF_PK bit.
*/
if (is_vsyscall_vaddr(address)) {
- if (emulate_vsyscall(hw_error_code, regs, address))
+ if (emulate_vsyscall(error_code, regs, address))
return;
}
#endif
@@ -1320,7 +1286,7 @@ void do_user_addr_fault(struct pt_regs *regs,
* Fault from code in kernel from
* which we do not expect faults.
*/
- bad_area_nosemaphore(regs, hw_error_code, address);
+ bad_area_nosemaphore(regs, error_code, address);
return;
}
retry:
@@ -1336,17 +1302,17 @@ void do_user_addr_fault(struct pt_regs *regs,
vma = find_vma(mm, address);
if (unlikely(!vma)) {
- bad_area(regs, hw_error_code, address);
+ bad_area(regs, error_code, address);
return;
}
if (likely(vma->vm_start <= address))
goto good_area;
if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
- bad_area(regs, hw_error_code, address);
+ bad_area(regs, error_code, address);
return;
}
if (unlikely(expand_stack(vma, address))) {
- bad_area(regs, hw_error_code, address);
+ bad_area(regs, error_code, address);
return;
}
@@ -1355,8 +1321,8 @@ void do_user_addr_fault(struct pt_regs *regs,
* we can handle it..
*/
good_area:
- if (unlikely(access_error(hw_error_code, vma))) {
- bad_area_access_error(regs, hw_error_code, address, vma);
+ if (unlikely(access_error(error_code, vma))) {
+ bad_area_access_error(regs, error_code, address, vma);
return;
}
@@ -1378,7 +1344,7 @@ void do_user_addr_fault(struct pt_regs *regs,
/* Quick path to respond to signals */
if (fault_signal_pending(fault, regs)) {
if (!user_mode(regs))
- no_context(regs, hw_error_code, address, SIGBUS,
+ no_context(regs, error_code, address, SIGBUS,
BUS_ADRERR);
return;
}
@@ -1395,10 +1361,37 @@ void do_user_addr_fault(struct pt_regs *regs,
}
mmap_read_unlock(mm);
- if (unlikely(fault & VM_FAULT_ERROR)) {
- mm_fault_error(regs, hw_error_code, address, fault);
+ if (likely(!(fault & VM_FAULT_ERROR)))
+ return;
+
+ if (fatal_signal_pending(current) && !(error_code & X86_PF_USER)) {
+ no_context(regs, error_code, address, 0, 0);
return;
}
+
+ if (fault & VM_FAULT_OOM) {
+ /* Kernel mode? Handle exceptions or die: */
+ if (!(error_code & X86_PF_USER)) {
+ no_context(regs, error_code, address,
+ SIGSEGV, SEGV_MAPERR);
+ return;
+ }
+
+ /*
+ * We ran out of memory, call the OOM killer, and return the
+ * userspace (which will retry the fault, or kill us if we got
+ * oom-killed):
+ */
+ pagefault_out_of_memory();
+ } else {
+ if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
+ VM_FAULT_HWPOISON_LARGE))
+ do_sigbus(regs, error_code, address, fault);
+ else if (fault & VM_FAULT_SIGSEGV)
+ bad_area_nosemaphore(regs, error_code, address);
+ else
+ BUG();
+ }
}
NOKPROBE_SYMBOL(do_user_addr_fault);
--
2.29.2
next prev parent reply other threads:[~2021-01-31 23:23 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-31 17:24 [PATCH 00/11] x86/fault: Cleanups and robustifications Andy Lutomirski
2021-01-31 17:24 ` [PATCH 01/11] x86/fault: Fix AMD erratum #91 errata fixup for user code Andy Lutomirski
2021-02-01 9:05 ` Christoph Hellwig
2021-02-01 20:31 ` Borislav Petkov
2021-01-31 17:24 ` Andy Lutomirski [this message]
2021-01-31 17:24 ` [PATCH 03/11] x86/fault/32: Move is_f00f_bug() do do_kern_addr_fault() Andy Lutomirski
2021-02-03 14:44 ` Borislav Petkov
2021-01-31 17:24 ` [PATCH 04/11] x86/fault: Document the locking in the fault_signal_pending() path Andy Lutomirski
2021-01-31 17:24 ` [PATCH 05/11] x86/fault: Correct a few user vs kernel checks wrt WRUSS Andy Lutomirski
2021-02-03 15:48 ` Borislav Petkov
2021-01-31 17:24 ` [PATCH 06/11] x86/fault: Improve kernel-executing-user-memory handling Andy Lutomirski
2021-02-01 9:08 ` Christoph Hellwig
2021-02-02 1:00 ` Andy Lutomirski
2021-02-03 16:01 ` Borislav Petkov
2021-02-03 16:23 ` Borislav Petkov
2021-01-31 17:24 ` [PATCH 07/11] x86/fault: Split the OOPS code out from no_context() Andy Lutomirski
2021-02-03 18:56 ` Borislav Petkov
2021-02-03 19:29 ` Andy Lutomirski
2021-02-03 19:46 ` Borislav Petkov
2021-02-09 20:09 ` Andy Lutomirski
2021-01-31 17:24 ` [PATCH 08/11] x86/fault: Bypass no_context() for implicit kernel faults from usermode Andy Lutomirski
2021-01-31 17:24 ` [PATCH 09/11] x86/fault: Rename no_context() to kernelmode_fixup_or_oops() Andy Lutomirski
2021-02-01 9:14 ` Christoph Hellwig
2021-02-02 1:01 ` Andy Lutomirski
2021-02-03 19:39 ` Borislav Petkov
2021-02-03 19:53 ` Andy Lutomirski
2021-02-03 20:07 ` Borislav Petkov
2021-02-03 20:14 ` Andy Lutomirski
2021-02-03 20:25 ` Borislav Petkov
2021-01-31 17:24 ` [PATCH 10/11] x86/fault: Don't run fixups for SMAP violations Andy Lutomirski
2021-02-03 19:50 ` Borislav Petkov
2021-01-31 17:24 ` [PATCH 11/11] x86/fault: Don't look for extable entries for SMEP violations Andy Lutomirski
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=c5adc9322511167403edd334ca760466535f582d.1612113550.git.luto@kernel.org \
--to=luto@kernel.org \
--cc=ast@kernel.org \
--cc=daniel@iogearbox.net \
--cc=dave.hansen@linux.intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mhiramat@kernel.org \
--cc=peterz@infradead.org \
--cc=x86@kernel.org \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).