From: Sasha Levin <sashal@kernel.org>
To: linux-kernel@vger.kernel.org, stable@vger.kernel.org
Cc: Peter Zijlstra <peterz@infradead.org>,
Andrea Parri <andrea.parri@amarulasolutions.com>,
Paul Burton <paul.burton@mips.com>,
Sasha Levin <sashal@kernel.org>,
linux-mips@vger.kernel.org
Subject: [PATCH AUTOSEL 5.2 13/42] mips/atomic: Fix smp_mb__{before,after}_atomic()
Date: Sun, 29 Sep 2019 13:32:12 -0400 [thread overview]
Message-ID: <20190929173244.8918-13-sashal@kernel.org> (raw)
In-Reply-To: <20190929173244.8918-1-sashal@kernel.org>
From: Peter Zijlstra <peterz@infradead.org>
[ Upstream commit 42344113ba7a1ed7b5654cd5270af0d5698d8521 ]
Recent probing at the Linux Kernel Memory Model uncovered a
'surprise'. Strongly ordered architectures where the atomic RmW
primitive implies full memory ordering and
smp_mb__{before,after}_atomic() are a simple barrier() (such as MIPS
without WEAK_REORDERING_BEYOND_LLSC) fail for:
*x = 1;
atomic_inc(u);
smp_mb__after_atomic();
r0 = *y;
Because, while the atomic_inc() implies memory order, it
(surprisingly) does not provide a compiler barrier. This then allows
the compiler to re-order like so:
atomic_inc(u);
*x = 1;
smp_mb__after_atomic();
r0 = *y;
Which the CPU is then allowed to re-order (under TSO rules) like:
atomic_inc(u);
r0 = *y;
*x = 1;
And this very much was not intended. Therefore strengthen the atomic
RmW ops to include a compiler barrier.
Reported-by: Andrea Parri <andrea.parri@amarulasolutions.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Paul Burton <paul.burton@mips.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
arch/mips/include/asm/atomic.h | 14 +++++------
arch/mips/include/asm/barrier.h | 12 ++++++++--
arch/mips/include/asm/bitops.h | 42 ++++++++++++++++++++-------------
arch/mips/include/asm/cmpxchg.h | 6 ++---
4 files changed, 45 insertions(+), 29 deletions(-)
diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
index c85405afba5ed..50cc2f0962e56 100644
--- a/arch/mips/include/asm/atomic.h
+++ b/arch/mips/include/asm/atomic.h
@@ -68,7 +68,7 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \
"\t" __scbeqz " %0, 1b \n" \
" .set pop \n" \
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -98,7 +98,7 @@ static __inline__ int atomic_##op##_return_relaxed(int i, atomic_t * v) \
" .set pop \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -132,7 +132,7 @@ static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \
" move %0, %1 \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -210,7 +210,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
" .set pop \n"
: "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter)
- : "Ir" (i));
+ : "Ir" (i) : __LLSC_CLOBBER);
} else {
unsigned long flags;
@@ -270,7 +270,7 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \
"\t" __scbeqz " %0, 1b \n" \
" .set pop \n" \
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -300,7 +300,7 @@ static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \
" .set pop \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
@@ -334,7 +334,7 @@ static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \
" .set pop \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
- : "Ir" (i)); \
+ : "Ir" (i) : __LLSC_CLOBBER); \
} else { \
unsigned long flags; \
\
diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h
index f9a6da96aae12..9228f73862205 100644
--- a/arch/mips/include/asm/barrier.h
+++ b/arch/mips/include/asm/barrier.h
@@ -211,14 +211,22 @@
#define __smp_wmb() barrier()
#endif
+/*
+ * When LL/SC does imply order, it must also be a compiler barrier to avoid the
+ * compiler from reordering where the CPU will not. When it does not imply
+ * order, the compiler is also free to reorder across the LL/SC loop and
+ * ordering will be done by smp_llsc_mb() and friends.
+ */
#if defined(CONFIG_WEAK_REORDERING_BEYOND_LLSC) && defined(CONFIG_SMP)
#define __WEAK_LLSC_MB " sync \n"
+#define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
+#define __LLSC_CLOBBER
#else
#define __WEAK_LLSC_MB " \n"
+#define smp_llsc_mb() do { } while (0)
+#define __LLSC_CLOBBER "memory"
#endif
-#define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
-
#ifdef CONFIG_CPU_CAVIUM_OCTEON
#define smp_mb__before_llsc() smp_wmb()
#define __smp_mb__before_llsc() __smp_wmb()
diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h
index 7bd35e5e2a9e4..985d6a02f9ea1 100644
--- a/arch/mips/include/asm/bitops.h
+++ b/arch/mips/include/asm/bitops.h
@@ -66,7 +66,8 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
" beqzl %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m));
+ : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)
+ : __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
loongson_llsc_mb();
@@ -76,7 +77,8 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
" " __INS "%0, %3, %2, 1 \n"
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (bit), "r" (~0));
+ : "ir" (bit), "r" (~0)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
} else if (kernel_uses_llsc) {
@@ -90,7 +92,8 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
" " __SC "%0, %1 \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit));
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
} else
__mips_set_bit(nr, addr);
@@ -122,7 +125,8 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
" beqzl %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (~(1UL << bit)));
+ : "ir" (~(1UL << bit))
+ : __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
loongson_llsc_mb();
@@ -132,7 +136,8 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
" " __INS "%0, $0, %2, 1 \n"
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (bit));
+ : "ir" (bit)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
} else if (kernel_uses_llsc) {
@@ -146,7 +151,8 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
" " __SC "%0, %1 \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (~(1UL << bit)));
+ : "ir" (~(1UL << bit))
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
} else
__mips_clear_bit(nr, addr);
@@ -192,7 +198,8 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
" beqzl %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit));
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -207,7 +214,8 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
" " __SC "%0, %1 \n"
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
- : "ir" (1UL << bit));
+ : "ir" (1UL << bit)
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
} else
__mips_change_bit(nr, addr);
@@ -244,7 +252,7 @@ static inline int test_and_set_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -260,7 +268,7 @@ static inline int test_and_set_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
@@ -301,7 +309,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+m" (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -317,7 +325,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
@@ -360,7 +368,7 @@ static inline int test_and_clear_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
} else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
@@ -375,7 +383,7 @@ static inline int test_and_clear_bit(unsigned long nr,
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "ir" (bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!temp));
#endif
} else if (kernel_uses_llsc) {
@@ -394,7 +402,7 @@ static inline int test_and_clear_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
@@ -437,7 +445,7 @@ static inline int test_and_change_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} else if (kernel_uses_llsc) {
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
unsigned long temp;
@@ -453,7 +461,7 @@ static inline int test_and_change_bit(unsigned long nr,
" .set pop \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
: "r" (1UL << bit)
- : "memory");
+ : __LLSC_CLOBBER);
} while (unlikely(!res));
res = temp & (1UL << bit);
diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h
index f5994a332673b..c8a47d18f6288 100644
--- a/arch/mips/include/asm/cmpxchg.h
+++ b/arch/mips/include/asm/cmpxchg.h
@@ -61,7 +61,7 @@ extern unsigned long __xchg_called_with_bad_pointer(void)
" .set pop \n" \
: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
: GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \
- : "memory"); \
+ : __LLSC_CLOBBER); \
} else { \
unsigned long __flags; \
\
@@ -134,8 +134,8 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
" .set pop \n" \
"2: \n" \
: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
- : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \
- : "memory"); \
+ : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \
+ : __LLSC_CLOBBER); \
loongson_llsc_mb(); \
} else { \
unsigned long __flags; \
--
2.20.1
next prev parent reply other threads:[~2019-09-29 17:33 UTC|newest]
Thread overview: 48+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-09-29 17:32 [PATCH AUTOSEL 5.2 01/42] MIPS: Ingenic: Disable broken BTB lookup optimization Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 02/42] clk: jz4740: Add TCU clock Sasha Levin
2019-09-29 17:32 ` Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 03/42] MIPS: Don't use bc_false uninitialized in __mm_isBranchInstr Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 04/42] MIPS: tlbex: Explicitly cast _PAGE_NO_EXEC to a boolean Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 05/42] i2c-cht-wc: Fix lockdep warning Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 06/42] mfd: intel-lpss: Remove D3cold delay Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 07/42] PCI: tegra: Fix OF node reference leak Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 08/42] HID: wacom: Fix several minor compiler warnings Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 09/42] mips/atomic: Fix loongson_llsc_mb() wreckage Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 10/42] PCI: pci-hyperv: Fix build errors on non-SYSFS config Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 11/42] PCI: layerscape: Add the bar_fixed_64bit property to the endpoint driver Sasha Levin
2019-09-29 17:32 ` Sasha Levin
2019-09-29 17:32 ` Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 12/42] livepatch: Nullify obj->mod in klp_module_coming()'s error path Sasha Levin
2019-09-29 17:32 ` Sasha Levin [this message]
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 14/42] ARM: 8898/1: mm: Don't treat faults reported from cache maintenance as writes Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 15/42] soundwire: intel: fix channel number reported by hardware Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 16/42] ARM: 8875/1: Kconfig: default to AEABI w/ Clang Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 17/42] MIPS: lantiq: update the clock alias' for the mainline PCIe PHY driver Sasha Levin
2019-09-29 17:39 ` Hauke Mehrtens
2019-10-05 23:00 ` Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 18/42] firmware: bcm47xx_nvram: Correct size_t printf format Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 19/42] rtc: snvs: fix possible race condition Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 20/42] rtc: pcf85363/pcf85263: fix regmap error in set_time Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 21/42] selinux: fix residual uses of current_security() for the SELinux blob Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 22/42] PCI: Add pci_info_ratelimited() to ratelimit PCI separately Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 23/42] HID: apple: Fix stuck function keys when using FN Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 24/42] PCI: rockchip: Propagate errors for optional regulators Sasha Levin
2019-09-29 17:32 ` Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 25/42] PCI: histb: " Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 26/42] PCI: imx6: " Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 27/42] PCI: exynos: Propagate errors for optional PHYs Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 28/42] security: smack: Fix possible null-pointer dereferences in smack_socket_sock_rcv_skb() Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 29/42] PCI: Use static const struct, not const static struct Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 30/42] ARM: 8905/1: Emit __gnu_mcount_nc when using Clang 10.0.0 or newer Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 31/42] ARM: 8903/1: ensure that usable memory in bank 0 starts from a PMD-aligned address Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 32/42] fat: work around race with userspace's read via blockdev while mounting Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 33/42] pktcdvd: remove warning on attempting to register non-passthrough dev Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 34/42] hypfs: Fix error number left in struct pointer member Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 35/42] crypto: hisilicon - Fix double free in sec_free_hw_sgl() Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 36/42] kbuild: clean compressed initramfs image Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 37/42] ocfs2: wait for recovering done after direct unlock request Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 38/42] kmemleak: increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE default to 16K Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 39/42] arm64: consider stack randomization for mmap base only when necessary Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 40/42] mips: properly account for stack randomization and stack guard gap Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 41/42] arm: " Sasha Levin
2019-09-29 17:32 ` [PATCH AUTOSEL 5.2 42/42] arm: use STACK_TOP when computing mmap base address Sasha Levin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190929173244.8918-13-sashal@kernel.org \
--to=sashal@kernel.org \
--cc=andrea.parri@amarulasolutions.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mips@vger.kernel.org \
--cc=paul.burton@mips.com \
--cc=peterz@infradead.org \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.