From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
stable@vger.kernel.org,
"Peter Zijlstra (Intel)" <peterz@infradead.org>,
Sean Christopherson <seanjc@google.com>,
Paolo Bonzini <pbonzini@redhat.com>
Subject: [PATCH 5.18 19/67] x86/uaccess: Implement macros for CMPXCHG on user addresses
Date: Fri, 3 Jun 2022 19:43:20 +0200 [thread overview]
Message-ID: <20220603173821.282670906@linuxfoundation.org> (raw)
In-Reply-To: <20220603173820.731531504@linuxfoundation.org>
From: Peter Zijlstra <peterz@infradead.org>
commit 989b5db215a2f22f89d730b607b071d964780f10 upstream.
Add support for CMPXCHG loops on userspace addresses. Provide both an
"unsafe" version for tight loops that do their own uaccess begin/end, as
well as a "safe" version for use cases where the CMPXCHG is not buried in
a loop, e.g. KVM will resume the guest instead of looping when emulation
of a guest atomic accesses fails the CMPXCHG.
Provide 8-byte versions for 32-bit kernels so that KVM can do CMPXCHG on
guest PAE PTEs, which are accessed via userspace addresses.
Guard the asm_volatile_goto() variation with CC_HAS_ASM_GOTO_TIED_OUTPUT,
the "+m" constraint fails on some compilers that otherwise support
CC_HAS_ASM_GOTO_OUTPUT.
Cc: stable@vger.kernel.org
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20220202004945.2540433-3-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
arch/x86/include/asm/uaccess.h | 142 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 142 insertions(+)
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -382,6 +382,103 @@ do { \
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
+ bool success; \
+ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
+ __typeof__(*(_ptr)) __old = *_old; \
+ __typeof__(*(_ptr)) __new = (_new); \
+ asm_volatile_goto("\n" \
+ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+ _ASM_EXTABLE_UA(1b, %l[label]) \
+ : CC_OUT(z) (success), \
+ [ptr] "+m" (*_ptr), \
+ [old] "+a" (__old) \
+ : [new] ltype (__new) \
+ : "memory" \
+ : label); \
+ if (unlikely(!success)) \
+ *_old = __old; \
+ likely(success); })
+
+#ifdef CONFIG_X86_32
+#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
+ bool success; \
+ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
+ __typeof__(*(_ptr)) __old = *_old; \
+ __typeof__(*(_ptr)) __new = (_new); \
+ asm_volatile_goto("\n" \
+ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
+ _ASM_EXTABLE_UA(1b, %l[label]) \
+ : CC_OUT(z) (success), \
+ "+A" (__old), \
+ [ptr] "+m" (*_ptr) \
+ : "b" ((u32)__new), \
+ "c" ((u32)((u64)__new >> 32)) \
+ : "memory" \
+ : label); \
+ if (unlikely(!success)) \
+ *_old = __old; \
+ likely(success); })
+#endif // CONFIG_X86_32
+#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
+#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
+ int __err = 0; \
+ bool success; \
+ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
+ __typeof__(*(_ptr)) __old = *_old; \
+ __typeof__(*(_ptr)) __new = (_new); \
+ asm volatile("\n" \
+ "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
+ CC_SET(z) \
+ "2:\n" \
+ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \
+ %[errout]) \
+ : CC_OUT(z) (success), \
+ [errout] "+r" (__err), \
+ [ptr] "+m" (*_ptr), \
+ [old] "+a" (__old) \
+ : [new] ltype (__new) \
+ : "memory", "cc"); \
+ if (unlikely(__err)) \
+ goto label; \
+ if (unlikely(!success)) \
+ *_old = __old; \
+ likely(success); })
+
+#ifdef CONFIG_X86_32
+/*
+ * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error.
+ * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
+ * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses
+ * both ESI and EDI for the memory operand, compilation will fail if the error
+ * is an input+output as there will be no register available for input.
+ */
+#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
+ int __result; \
+ __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
+ __typeof__(*(_ptr)) __old = *_old; \
+ __typeof__(*(_ptr)) __new = (_new); \
+ asm volatile("\n" \
+ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
+ "mov $0, %%ecx\n\t" \
+ "setz %%cl\n" \
+ "2:\n" \
+ _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \
+ : [result]"=c" (__result), \
+ "+A" (__old), \
+ [ptr] "+m" (*_ptr) \
+ : "b" ((u32)__new), \
+ "c" ((u32)((u64)__new >> 32)) \
+ : "memory", "cc"); \
+ if (unlikely(__result < 0)) \
+ goto label; \
+ if (unlikely(!__result)) \
+ *_old = __old; \
+ likely(__result); })
+#endif // CONFIG_X86_32
+#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
+
/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x))
@@ -474,6 +571,51 @@ do { \
} while (0)
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+extern void __try_cmpxchg_user_wrong_size(void);
+
+#ifndef CONFIG_X86_32
+#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \
+ __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
+#endif
+
+/*
+ * Force the pointer to u<size> to match the size expected by the asm helper.
+ * clang/LLVM compiles all cases and only discards the unused paths after
+ * processing errors, which breaks i386 if the pointer is an 8-byte value.
+ */
+#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
+ bool __ret; \
+ __chk_user_ptr(_ptr); \
+ switch (sizeof(*(_ptr))) { \
+ case 1: __ret = __try_cmpxchg_user_asm("b", "q", \
+ (__force u8 *)(_ptr), (_oldp), \
+ (_nval), _label); \
+ break; \
+ case 2: __ret = __try_cmpxchg_user_asm("w", "r", \
+ (__force u16 *)(_ptr), (_oldp), \
+ (_nval), _label); \
+ break; \
+ case 4: __ret = __try_cmpxchg_user_asm("l", "r", \
+ (__force u32 *)(_ptr), (_oldp), \
+ (_nval), _label); \
+ break; \
+ case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
+ (_nval), _label); \
+ break; \
+ default: __try_cmpxchg_user_wrong_size(); \
+ } \
+ __ret; })
+
+/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
+#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
+ int __ret = -EFAULT; \
+ __uaccess_begin_nospec(); \
+ __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \
+_label: \
+ __uaccess_end(); \
+ __ret; \
+ })
+
/*
* We want the unsafe accessors to always be inlined and use
* the error labels - thus the macro games.
next prev parent reply other threads:[~2022-06-03 18:16 UTC|newest]
Thread overview: 75+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-03 17:43 [PATCH 5.18 00/67] 5.18.2-rc1 review Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 01/67] netfilter: nf_tables: disallow non-stateful expression in sets earlier Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 02/67] i2c: ismt: prevent memory corruption in ismt_access() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 03/67] assoc_array: Fix BUG_ON during garbage collect Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 04/67] pipe: make poll_usage boolean and annotate its access Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 05/67] pipe: Fix missing lock in pipe_resize_ring() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 06/67] net: ipa: compute proper aggregation limit Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 07/67] drm/i915: Fix -Wstringop-overflow warning in call to intel_read_wm_latency() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 08/67] exfat: check if cluster num is valid Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 09/67] exfat: fix referencing wrong parent directory information after renaming Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 10/67] netfilter: nft_limit: Clone packet limits cost value Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 11/67] netfilter: nf_tables: sanitize nft_set_desc_concat_parse() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 12/67] netfilter: nf_tables: hold mutex on netns pre_exit path Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 13/67] netfilter: nf_tables: double hook unregistration in netns path Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 14/67] netfilter: conntrack: re-fetch conntrack after insertion Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 15/67] KVM: PPC: Book3S HV: fix incorrect NULL check on list iterator Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 16/67] x86/fpu: KVM: Set the base guest FPU uABI size to sizeof(struct kvm_xsave) Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 17/67] x86/kvm: Alloc dummy async #PF token outside of raw spinlock Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 18/67] x86, kvm: use correct GFP flags for preemption disabled Greg Kroah-Hartman
2022-06-03 17:43 ` Greg Kroah-Hartman [this message]
2022-06-03 17:43 ` [PATCH 5.18 20/67] KVM: x86: Use __try_cmpxchg_user() to update guest PTE A/D bits Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 21/67] KVM: x86: Use __try_cmpxchg_user() to emulate atomic accesses Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 22/67] KVM: x86: fix typo in __try_cmpxchg_user causing non-atomicness Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 23/67] KVM: x86: avoid calling x86 emulator without a decoded instruction Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 24/67] KVM: x86: avoid loading a vCPU after .vm_destroy was called Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 25/67] KVM: x86: Fix the intel_pt PMI handling wrongly considered from guest Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 26/67] KVM: x86: Drop WARNs that assert a triple fault never "escapes" from L2 Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 27/67] KVM: x86/mmu: Dont rebuild page when the page is synced and no tlb flushing is required Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 28/67] KVM: SVM: Use kzalloc for sev ioctl interfaces to prevent kernel data leak Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 29/67] crypto: caam - fix i.MX6SX entropy delay value Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 30/67] crypto: ecrdsa - Fix incorrect use of vli_cmp Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 31/67] crypto: qat - rework the VF2PF interrupt handling logic Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 32/67] zsmalloc: fix races between asynchronous zspage free and page migration Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 33/67] tools/memory-model/README: Update klitmus7 compat table Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 34/67] ALSA: usb-audio: Workaround for clock setup on TEAC devices Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 35/67] ALSA: usb-audio: Add missing ep_idx in fixed EP quirks Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 36/67] ALSA: usb-audio: Configure sync endpoints before data Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 37/67] Bluetooth: hci_qca: Use del_timer_sync() before freeing Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 38/67] ARM: dts: s5pv210: Correct interrupt name for bluetooth in Aries Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 39/67] dm integrity: fix error code in dm_integrity_ctr() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 40/67] dm crypt: make printing of the key constant-time Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 41/67] dm stats: add cond_resched when looping over entries Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 42/67] dm verity: set DM_TARGET_IMMUTABLE feature flag Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 43/67] raid5: introduce MD_BROKEN Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 44/67] fs/ntfs3: validate BOOT sectors_per_clusters Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 45/67] HID: multitouch: Add support for Google Whiskers Touchpad Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 46/67] HID: multitouch: add quirks to enable Lenovo X12 trackpoint Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 47/67] x86/sgx: Disconnect backing page references from dirty status Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 48/67] x86/sgx: Mark PCMD page as dirty when modifying contents Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 49/67] x86/sgx: Obtain backing storage page with enclave mutex held Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 50/67] x86/sgx: Fix race between reclaimer and page fault handler Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 51/67] x86/sgx: Ensure no data in PCMD page after truncate Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 52/67] media: i2c: imx412: Fix reset GPIO polarity Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 53/67] media: i2c: imx412: Fix power_off ordering Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 54/67] tpm: Fix buffer access in tpm2_get_tpm_pt() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 55/67] tpm: ibmvtpm: Correct the return value in tpm_ibmvtpm_probe() Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 56/67] docs: submitting-patches: Fix crossref to The canonical patch format Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 57/67] NFS: Memory allocation failures are not server fatal errors Greg Kroah-Hartman
2022-06-03 17:43 ` [PATCH 5.18 58/67] NFSD: Fix possible sleep during nfsd4_release_lockowner() Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 59/67] bpf: Fill new bpf_prog_pack with illegal instructions Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 60/67] bpf: Fix potential array overflow in bpf_trampoline_get_progs() Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 61/67] bpf: Fix combination of jit blinding and pointers to bpf subprogs Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 62/67] bpf: Enlarge offset check value to INT_MAX in bpf_skb_{load,store}_bytes Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 63/67] bpf: Fix usage of trace RCU in local storage Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 64/67] bpf: Fix excessive memory allocation in stack_map_alloc() Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 65/67] bpf: Reject writes for PTR_TO_MAP_KEY in check_helper_mem_access Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 66/67] bpf: Check PTR_TO_MEM | MEM_RDONLY " Greg Kroah-Hartman
2022-06-03 17:44 ` [PATCH 5.18 67/67] bpf: Do write access check for kfunc and global func Greg Kroah-Hartman
2022-06-03 23:00 ` [PATCH 5.18 00/67] 5.18.2-rc1 review Justin Forbes
2022-06-04 6:25 ` Ron Economos
2022-06-04 16:45 ` Naresh Kamboju
2022-06-04 18:56 ` Guenter Roeck
2022-06-05 2:28 ` Rudi Heitbaum
2022-06-05 7:02 ` Bagas Sanjaya
2022-06-05 7:38 ` Greg Kroah-Hartman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220603173821.282670906@linuxfoundation.org \
--to=gregkh@linuxfoundation.org \
--cc=linux-kernel@vger.kernel.org \
--cc=pbonzini@redhat.com \
--cc=peterz@infradead.org \
--cc=seanjc@google.com \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).