From: Sasha Levin <sashal@kernel.org>
To: linux-kernel@vger.kernel.org, stable@vger.kernel.org
Cc: Peter Zijlstra <peterz@infradead.org>,
Andrea Parri <andrea.parri@amarulasolutions.com>,
Paul Burton <paul.burton@mips.com>,
Sasha Levin <sashal@kernel.org>,
linux-mips@vger.kernel.org
Subject: [PATCH AUTOSEL 5.2 13/42] mips/atomic: Fix smp_mb__{before,after}_atomic()
Date: Sun, 29 Sep 2019 13:32:12 -0400 [thread overview]
Message-ID: <20190929173244.8918-13-sashal@kernel.org> (raw)
In-Reply-To: <20190929173244.8918-1-sashal@kernel.org>
From: Peter Zijlstra <peterz@infradead.org>
[ Upstream commit 42344113ba7a1ed7b5654cd5270af0d5698d8521 ]
Recent probing at the Linux Kernel Memory Model uncovered a
'surprise'. Strongly ordered architectures where the atomic RmW
primitive implies full memory ordering and
smp_mb__{before,after}_atomic() are a simple barrier() (such as MIPS
without WEAK_REORDERING_BEYOND_LLSC) fail for:
*x = 1;
atomic_inc(u);
smp_mb__after_atomic();
r0 = *y;
Because, while the atomic_inc() implies memory order, it
(surprisingly) does not provide a compiler barrier. This then allows
the compiler to re-order like so:
atomic_inc(u);
*x = 1;
smp_mb__after_atomic();
r0 = *y;
Which the CPU is then allowed to re-order (under TSO rules) like:
atomic_inc(u);
r0 = *y;
*x = 1;
And this very much was not intended. Therefore strengthen the atomic
RmW ops to include a compiler barrier.
Reported-by: Andrea Parri <andrea.parri@amarulasolutions.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Paul Burton <paul.burton@mips.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
arch/mips/include/asm/atomic.h | 14 +++++------
arch/mips/include/asm/barrier.h | 12 ++++++++--
arch/mips/include/asm/bitops.h | 42 ++++++++++++++++++++-------------
arch/mips/include/asm/cmpxchg.h | 6 ++---
4 files changed, 45 insertions(+), 29 deletions(-)
diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
index c85405afba5ed..50cc2f0962e56 100644
--- a/arch/mips/include/asm/atomic.h
+++ b/arch/mips/include/asm/atomic.h
@@ -68,7 +68,7 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \
"\t" __scbeqz " %0, 1b \n" \
" .set pop \n" \
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -98,7 +98,7 @@ static __inline__ int atomic_##op##_return_relaxed(int i, atomic_t * v) \
" .set pop \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -132,7 +132,7 @@ static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \
" move %0, %1 \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -210,7 +210,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
" .set pop \n"
: "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter)
- : "Ir" (i));
+ : "Ir" (i) : __LLSC_CLOBBER);
} else {
unsigned long flags;
@@ -270,7 +270,7 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \
"\t" __scbeqz " %0, 1b \n" \
" .set pop \n" \
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -300,7 +300,7 @@ static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \
" .set pop \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -334,7 +334,7 @@ static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \
" .set pop \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h
index f9a6da96aae12..9228f73862205 100644
--- a/arch/mips/include/asm/barrier.h
+++ b/arch/mips/include/asm/barrier.h
@@ -211,14 +211,22 @@
#define __smp_wmb() barrier()
#endif
+/*
+ * When LL/SC does imply order, it must also be a compiler barrier to avoid the
+ * compiler from reordering where the CPU will not. When it does not imply
+ * order, the compiler is also free to reorder across the LL/SC loop and
+ * ordering will be done by smp_llsc_mb() and friends.
+ */
#if defined(CONFIG_WEAK_REORDERING_BEYOND_LLSC) && defined(CONFIG_SMP)
#define __WEAK_LLSC_MB " sync \n"
+#define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
+#define __LLSC_CLOBBER
#else
#define __WEAK_LLSC_MB " \n"
+#define smp_llsc_mb() do { } while (0)
+#define __LLSC_CLOBBER "memory"
#endif
-#define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
-
#ifdef CONFIG_CPU_CAVIUM_OCTEON
#define smp_mb__before_llsc() smp_wmb()
#define __smp_mb__before_llsc() __smp_wmb()
diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h
index 7bd35e5e2a9e4..985d6a02f9ea1 100644
--- a/arch/mips/include/asm/bitops.h
+++ b/arch/mips/include/asm/bitops.h
@@ -66,7 +66,8 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
" beqzl %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m));
+ : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)
+ : __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
loongson_llsc_mb();
@@ -76,7 +77,8 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
" " __INS "%0, %3, %2, 1 \n"
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (bit), "r" (~0));
+ : "ir" (bit), "r" (~0)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
} else if (kernel_uses_llsc) {
@@ -90,7 +92,8 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
" " __SC "%0, %1 \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit));
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
} else
__mips_set_bit(nr, addr);
@@ -122,7 +125,8 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
" beqzl %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (~(1UL << bit)));
+ : "ir" (~(1UL << bit))
+ : __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
loongson_llsc_mb();
@@ -132,7 +136,8 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
" " __INS "%0, $0, %2, 1 \n"
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (bit));
+ : "ir" (bit)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
} else if (kernel_uses_llsc) {
@@ -146,7 +151,8 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
" " __SC "%0, %1 \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (~(1UL << bit)));
+ : "ir" (~(1UL << bit))
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
} else
__mips_clear_bit(nr, addr);
@@ -192,7 +198,8 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
" beqzl %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit));
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -207,7 +214,8 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
" " __SC "%0, %1 \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit));
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
} else
__mips_change_bit(nr, addr);
@@ -244,7 +252,7 @@ static inline int test_and_set_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -260,7 +268,7 @@ static inline int test_and_set_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
@@ -301,7 +309,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+m" (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -317,7 +325,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
@@ -360,7 +368,7 @@ static inline int test_and_clear_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
} else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
@@ -375,7 +383,7 @@ static inline int test_and_clear_bit(unsigned long nr,
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "ir" (bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif
} else if (kernel_uses_llsc) {
@@ -394,7 +402,7 @@ static inline int test_and_clear_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
@@ -437,7 +445,7 @@ static inline int test_and_change_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -453,7 +461,7 @@ static inline int test_and_change_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h
index f5994a332673b..c8a47d18f6288 100644
--- a/arch/mips/include/asm/cmpxchg.h
+++ b/arch/mips/include/asm/cmpxchg.h
@@ -61,7 +61,7 @@ extern unsigned long __xchg_called_with_bad_pointer(void)
" .set pop \n" \
: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
: GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \
- : "memory"); \
+ : __LLSC_CLOBBER); \
} else { \
unsigned long __flags; \
\
@@ -134,8 +134,8 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
" .set pop \n" \
"2: \n" \
: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
- : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \
- : "memory"); \
+ : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \
+ : __LLSC_CLOBBER); \
loongson_llsc_mb(); \
} else { \
unsigned long __flags; \
--
2.20.1
next prev parent reply other threads:[~2019-09-29 17:33 UTC|newest]
Thread overview: 44+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-09-29 17:32 [PATCH AUTOSEL 5.2 01/42] MIPS: Ingenic: Disable broken BTB lookup optimization Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 02/42] clk: jz4740: Add TCU clock Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 03/42] MIPS: Don't use bc_false uninitialized in __mm_isBranchInstr Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 04/42] MIPS: tlbex: Explicitly cast _PAGE_NO_EXEC to a boolean Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 05/42] i2c-cht-wc: Fix lockdep warning Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 06/42] mfd: intel-lpss: Remove D3cold delay Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 07/42] PCI: tegra: Fix OF node reference leak Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 08/42] HID: wacom: Fix several minor compiler warnings Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 09/42] mips/atomic: Fix loongson_llsc_mb() wreckage Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 10/42] PCI: pci-hyperv: Fix build errors on non-SYSFS config Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 11/42] PCI: layerscape: Add the bar_fixed_64bit property to the endpoint driver Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 12/42] livepatch: Nullify obj->mod in klp_module_coming()'s error path Sasha Levin
2019-09-29 17:32 ` Sasha Levin [this message]
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 14/42] ARM: 8898/1: mm: Don't treat faults reported from cache maintenance as writes Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 15/42] soundwire: intel: fix channel number reported by hardware Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 16/42] ARM: 8875/1: Kconfig: default to AEABI w/ Clang Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 17/42] MIPS: lantiq: update the clock alias' for the mainline PCIe PHY driver Sasha Levin
2019-09-29 17:39 ` Hauke Mehrtens
2019-10-05 23:00 ` Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 18/42] firmware: bcm47xx_nvram: Correct size_t printf format Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 19/42] rtc: snvs: fix possible race condition Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 20/42] rtc: pcf85363/pcf85263: fix regmap error in set_time Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 21/42] selinux: fix residual uses of current_security() for the SELinux blob Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 22/42] PCI: Add pci_info_ratelimited() to ratelimit PCI separately Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 23/42] HID: apple: Fix stuck function keys when using FN Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 24/42] PCI: rockchip: Propagate errors for optional regulators Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 25/42] PCI: histb: " Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 26/42] PCI: imx6: " Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 27/42] PCI: exynos: Propagate errors for optional PHYs Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 28/42] security: smack: Fix possible null-pointer dereferences in smack_socket_sock_rcv_skb() Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 29/42] PCI: Use static const struct, not const static struct Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 30/42] ARM: 8905/1: Emit __gnu_mcount_nc when using Clang 10.0.0 or newer Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 31/42] ARM: 8903/1: ensure that usable memory in bank 0 starts from a PMD-aligned address Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 32/42] fat: work around race with userspace's read via blockdev while mounting Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 33/42] pktcdvd: remove warning on attempting to register non-passthrough dev Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 34/42] hypfs: Fix error number left in struct pointer member Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 35/42] crypto: hisilicon - Fix double free in sec_free_hw_sgl() Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 36/42] kbuild: clean compressed initramfs image Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 37/42] ocfs2: wait for recovering done after direct unlock request Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 38/42] kmemleak: increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE default to 16K Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 39/42] arm64: consider stack randomization for mmap base only when necessary Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 40/42] mips: properly account for stack randomization and stack guard gap Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 41/42] arm: " Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 42/42] arm: use STACK_TOP when computing mmap base address Sasha Levin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190929173244.8918-13-sashal@kernel.org \
--to=sashal@kernel.org \
--cc=andrea.parri@amarulasolutions.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mips@vger.kernel.org \
--cc=paul.burton@mips.com \
--cc=peterz@infradead.org \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).