From: Marco Elver <elver@google.com>
To: elver@google.com, "Paul E. McKenney" <paulmck@kernel.org>
Cc: Alexander Potapenko <glider@google.com>,
Boqun Feng <boqun.feng@gmail.com>, Borislav Petkov <bp@alien8.de>,
Dmitry Vyukov <dvyukov@google.com>,
Ingo Molnar <mingo@kernel.org>,
Josh Poimboeuf <jpoimboe@redhat.com>,
Mark Rutland <mark.rutland@arm.com>,
Peter Zijlstra <peterz@infradead.org>,
Thomas Gleixner <tglx@linutronix.de>,
Waiman Long <longman@redhat.com>, Will Deacon <will@kernel.org>,
kasan-dev@googlegroups.com, linux-arch@vger.kernel.org,
linux-doc@vger.kernel.org, linux-kbuild@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-mm@kvack.org, x86@kernel.org
Subject: [PATCH v2 05/23] kcsan: Add core memory barrier instrumentation functions
Date: Thu, 18 Nov 2021 09:10:09 +0100 [thread overview]
Message-ID: <20211118081027.3175699-6-elver@google.com> (raw)
In-Reply-To: <20211118081027.3175699-1-elver@google.com>
Add the core memory barrier instrumentation functions. These invalidate
the current in-flight reordered access based on the rules for the
respective barrier types and in-flight access type.
Signed-off-by: Marco Elver <elver@google.com>
---
v2:
* Rename kcsan_atomic_release() to kcsan_atomic_builtin_memorder() to
avoid confusion.
---
include/linux/kcsan-checks.h | 41 ++++++++++++++++++++++++++++++++++--
kernel/kcsan/core.c | 36 +++++++++++++++++++++++++++++++
2 files changed, 75 insertions(+), 2 deletions(-)
diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h
index a1c6a89fde71..c9e7c39a7d7b 100644
--- a/include/linux/kcsan-checks.h
+++ b/include/linux/kcsan-checks.h
@@ -36,6 +36,26 @@
*/
void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
+/**
+ * __kcsan_mb - full memory barrier instrumentation
+ */
+void __kcsan_mb(void);
+
+/**
+ * __kcsan_wmb - write memory barrier instrumentation
+ */
+void __kcsan_wmb(void);
+
+/**
+ * __kcsan_rmb - read memory barrier instrumentation
+ */
+void __kcsan_rmb(void);
+
+/**
+ * __kcsan_release - release barrier instrumentation
+ */
+void __kcsan_release(void);
+
/**
* kcsan_disable_current - disable KCSAN for the current context
*
@@ -159,6 +179,10 @@ void kcsan_end_scoped_access(struct kcsan_scoped_access *sa);
static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
int type) { }
+static inline void __kcsan_mb(void) { }
+static inline void __kcsan_wmb(void) { }
+static inline void __kcsan_rmb(void) { }
+static inline void __kcsan_release(void) { }
static inline void kcsan_disable_current(void) { }
static inline void kcsan_enable_current(void) { }
static inline void kcsan_enable_current_nowarn(void) { }
@@ -191,12 +215,25 @@ static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { }
*/
#define __kcsan_disable_current kcsan_disable_current
#define __kcsan_enable_current kcsan_enable_current_nowarn
-#else
+#else /* __SANITIZE_THREAD__ */
static inline void kcsan_check_access(const volatile void *ptr, size_t size,
int type) { }
static inline void __kcsan_enable_current(void) { }
static inline void __kcsan_disable_current(void) { }
-#endif
+#endif /* __SANITIZE_THREAD__ */
+
+#if defined(CONFIG_KCSAN_WEAK_MEMORY) && \
+ (defined(__SANITIZE_THREAD__) || defined(__KCSAN_INSTRUMENT_BARRIERS__))
+#define kcsan_mb __kcsan_mb
+#define kcsan_wmb __kcsan_wmb
+#define kcsan_rmb __kcsan_rmb
+#define kcsan_release __kcsan_release
+#else /* CONFIG_KCSAN_WEAK_MEMORY && (__SANITIZE_THREAD__ || __KCSAN_INSTRUMENT_BARRIERS__) */
+static inline void kcsan_mb(void) { }
+static inline void kcsan_wmb(void) { }
+static inline void kcsan_rmb(void) { }
+static inline void kcsan_release(void) { }
+#endif /* CONFIG_KCSAN_WEAK_MEMORY && (__SANITIZE_THREAD__ || __KCSAN_INSTRUMENT_BARRIERS__) */
/**
* __kcsan_check_read - check regular read access for races
diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c
index 24d82baa807d..840ed8e35f75 100644
--- a/kernel/kcsan/core.c
+++ b/kernel/kcsan/core.c
@@ -955,6 +955,28 @@ void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
}
EXPORT_SYMBOL(__kcsan_check_access);
+#define DEFINE_MEMORY_BARRIER(name, order_before_cond) \
+ kcsan_noinstr void __kcsan_##name(void) \
+ { \
+ struct kcsan_scoped_access *sa; \
+ if (within_noinstr(_RET_IP_)) \
+ return; \
+ instrumentation_begin(); \
+ sa = get_reorder_access(get_ctx()); \
+ if (!sa) \
+ goto out; \
+ if (order_before_cond) \
+ sa->size = 0; \
+ out: \
+ instrumentation_end(); \
+ } \
+ EXPORT_SYMBOL(__kcsan_##name)
+
+DEFINE_MEMORY_BARRIER(mb, true);
+DEFINE_MEMORY_BARRIER(wmb, sa->type & (KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND));
+DEFINE_MEMORY_BARRIER(rmb, !(sa->type & KCSAN_ACCESS_WRITE) || (sa->type & KCSAN_ACCESS_COMPOUND));
+DEFINE_MEMORY_BARRIER(release, true);
+
/*
* KCSAN uses the same instrumentation that is emitted by supported compilers
* for ThreadSanitizer (TSAN).
@@ -1143,10 +1165,19 @@ EXPORT_SYMBOL(__tsan_init);
* functions, whose job is to also execute the operation itself.
*/
+static __always_inline void kcsan_atomic_builtin_memorder(int memorder)
+{
+ if (memorder == __ATOMIC_RELEASE ||
+ memorder == __ATOMIC_SEQ_CST ||
+ memorder == __ATOMIC_ACQ_REL)
+ __kcsan_release();
+}
+
#define DEFINE_TSAN_ATOMIC_LOAD_STORE(bits) \
u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder); \
u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder) \
{ \
+ kcsan_atomic_builtin_memorder(memorder); \
if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
check_access(ptr, bits / BITS_PER_BYTE, KCSAN_ACCESS_ATOMIC, _RET_IP_); \
} \
@@ -1156,6 +1187,7 @@ EXPORT_SYMBOL(__tsan_init);
void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder); \
void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder) \
{ \
+ kcsan_atomic_builtin_memorder(memorder); \
if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
check_access(ptr, bits / BITS_PER_BYTE, \
KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC, _RET_IP_); \
@@ -1168,6 +1200,7 @@ EXPORT_SYMBOL(__tsan_init);
u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder); \
u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder) \
{ \
+ kcsan_atomic_builtin_memorder(memorder); \
if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
check_access(ptr, bits / BITS_PER_BYTE, \
KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
@@ -1200,6 +1233,7 @@ EXPORT_SYMBOL(__tsan_init);
int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp, \
u##bits val, int mo, int fail_mo) \
{ \
+ kcsan_atomic_builtin_memorder(mo); \
if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
check_access(ptr, bits / BITS_PER_BYTE, \
KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
@@ -1215,6 +1249,7 @@ EXPORT_SYMBOL(__tsan_init);
u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \
int mo, int fail_mo) \
{ \
+ kcsan_atomic_builtin_memorder(mo); \
if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
check_access(ptr, bits / BITS_PER_BYTE, \
KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
@@ -1246,6 +1281,7 @@ DEFINE_TSAN_ATOMIC_OPS(64);
void __tsan_atomic_thread_fence(int memorder);
void __tsan_atomic_thread_fence(int memorder)
{
+ kcsan_atomic_builtin_memorder(memorder);
__atomic_thread_fence(memorder);
}
EXPORT_SYMBOL(__tsan_atomic_thread_fence);
--
2.34.0.rc2.393.gf8c9666880-goog
next prev parent reply other threads:[~2021-11-18 8:11 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-18 8:10 [PATCH v2 00/23] kcsan: Support detecting a subset of missing memory barriers Marco Elver
2021-11-18 8:10 ` [PATCH v2 01/23] kcsan: Refactor reading of instrumented memory Marco Elver
2021-11-18 11:08 ` Mark Rutland
2021-11-18 8:10 ` [PATCH v2 02/23] kcsan: Remove redundant zero-initialization of globals Marco Elver
2021-11-18 11:09 ` Mark Rutland
2021-11-18 8:10 ` [PATCH v2 03/23] kcsan: Avoid checking scoped accesses from nested contexts Marco Elver
2021-11-29 8:47 ` Boqun Feng
2021-11-29 10:57 ` Marco Elver
2021-11-29 14:26 ` Boqun Feng
2021-11-29 14:42 ` Marco Elver
2021-11-18 8:10 ` [PATCH v2 04/23] kcsan: Add core support for a subset of weak memory modeling Marco Elver
2021-11-18 8:10 ` Marco Elver [this message]
2021-11-18 8:10 ` [PATCH v2 06/23] kcsan, kbuild: Add option for barrier instrumentation only Marco Elver
2021-11-18 8:10 ` [PATCH v2 07/23] kcsan: Call scoped accesses reordered in reports Marco Elver
2021-11-18 8:10 ` [PATCH v2 08/23] kcsan: Show location access was reordered to Marco Elver
2021-11-18 8:10 ` [PATCH v2 09/23] kcsan: Document modeling of weak memory Marco Elver
2021-11-18 8:10 ` [PATCH v2 10/23] kcsan: test: Match reordered or normal accesses Marco Elver
2021-11-18 8:10 ` [PATCH v2 11/23] kcsan: test: Add test cases for memory barrier instrumentation Marco Elver
2021-11-18 8:10 ` [PATCH v2 12/23] kcsan: Ignore GCC 11+ warnings about TSan runtime support Marco Elver
2021-11-18 8:10 ` [PATCH v2 13/23] kcsan: selftest: Add test case to check memory barrier instrumentation Marco Elver
2021-11-18 8:10 ` [PATCH v2 14/23] locking/barriers, kcsan: Add instrumentation for barriers Marco Elver
2021-11-18 8:10 ` [PATCH v2 15/23] locking/barriers, kcsan: Support generic instrumentation Marco Elver
2021-11-18 8:10 ` [PATCH v2 16/23] locking/atomics, kcsan: Add instrumentation for barriers Marco Elver
2021-11-18 8:10 ` [PATCH v2 17/23] asm-generic/bitops, " Marco Elver
2021-11-18 8:10 ` [PATCH v2 18/23] x86/barriers, kcsan: Use generic instrumentation for non-smp barriers Marco Elver
2021-11-18 8:10 ` [PATCH v2 19/23] x86/qspinlock, kcsan: Instrument barrier of pv_queued_spin_unlock() Marco Elver
2021-11-18 8:10 ` [PATCH v2 20/23] mm, kcsan: Enable barrier instrumentation Marco Elver
2021-11-18 8:10 ` [PATCH v2 21/23] sched, kcsan: Enable memory " Marco Elver
2021-11-18 8:10 ` [PATCH v2 22/23] objtool, kcsan: Add memory barrier instrumentation to whitelist Marco Elver
2021-11-18 8:10 ` [PATCH v2 23/23] objtool, kcsan: Remove memory barrier instrumentation from noinstr Marco Elver
2021-11-19 20:31 ` Josh Poimboeuf
2021-11-19 21:31 ` Marco Elver
2021-11-23 11:29 ` Marco Elver
2021-11-24 17:53 ` Josh Poimboeuf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211118081027.3175699-6-elver@google.com \
--to=elver@google.com \
--cc=boqun.feng@gmail.com \
--cc=bp@alien8.de \
--cc=dvyukov@google.com \
--cc=glider@google.com \
--cc=jpoimboe@redhat.com \
--cc=kasan-dev@googlegroups.com \
--cc=linux-arch@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kbuild@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=longman@redhat.com \
--cc=mark.rutland@arm.com \
--cc=mingo@kernel.org \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=tglx@linutronix.de \
--cc=will@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).