From: paulmck@kernel.org
To: linux-kernel@vger.kernel.org, kasan-dev@googlegroups.com,
kernel-team@fb.com, mingo@kernel.org
Cc: elver@google.com, andreyknvl@google.com, glider@google.com,
dvyukov@google.com, cai@lca.pw, boqun.feng@gmail.com,
"Paul E . McKenney" <paulmck@kernel.org>
Subject: [PATCH v4 tip/core/rcu 03/15] kcsan: Add current->state to implicitly atomic accesses
Date: Wed, 15 Apr 2020 11:33:59 -0700 [thread overview]
Message-ID: <20200415183411.12368-3-paulmck@kernel.org> (raw)
In-Reply-To: <20200415183343.GA12265@paulmck-ThinkPad-P72>
From: Marco Elver <elver@google.com>
Add volatile current->state to list of implicitly atomic accesses. This
is in preparation to eventually enable KCSAN on kernel/sched (which
currently still has KCSAN_SANITIZE := n).
Since accesses that match the special check in atomic.h are rare, it
makes more sense to move this check to the slow-path, avoiding the
additional compare in the fast-path. With the microbenchmark, a speedup
of ~6% is measured.
Signed-off-by: Marco Elver <elver@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
kernel/kcsan/atomic.h | 21 +++++++--------------
kernel/kcsan/core.c | 22 +++++++++++++++-------
kernel/kcsan/debugfs.c | 27 ++++++++++++++++++---------
3 files changed, 40 insertions(+), 30 deletions(-)
diff --git a/kernel/kcsan/atomic.h b/kernel/kcsan/atomic.h
index a9c1930..be9e625 100644
--- a/kernel/kcsan/atomic.h
+++ b/kernel/kcsan/atomic.h
@@ -4,24 +4,17 @@
#define _KERNEL_KCSAN_ATOMIC_H
#include <linux/jiffies.h>
+#include <linux/sched.h>
/*
- * Helper that returns true if access to @ptr should be considered an atomic
- * access, even though it is not explicitly atomic.
- *
- * List all volatile globals that have been observed in races, to suppress
- * data race reports between accesses to these variables.
- *
- * For now, we assume that volatile accesses of globals are as strong as atomic
- * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not
- * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more
- * than cast to volatile. Eventually, we hope to be able to remove this
- * function.
+ * Special rules for certain memory where concurrent conflicting accesses are
+ * common, however, the current convention is to not mark them; returns true if
+ * access to @ptr should be considered atomic. Called from slow-path.
*/
-static __always_inline bool kcsan_is_atomic(const volatile void *ptr)
+static bool kcsan_is_atomic_special(const volatile void *ptr)
{
- /* only jiffies for now */
- return ptr == &jiffies;
+ /* volatile globals that have been observed in data races. */
+ return ptr == &jiffies || ptr == ¤t->state;
}
#endif /* _KERNEL_KCSAN_ATOMIC_H */
diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c
index 065615d..eb30ecd 100644
--- a/kernel/kcsan/core.c
+++ b/kernel/kcsan/core.c
@@ -188,12 +188,13 @@ static __always_inline struct kcsan_ctx *get_ctx(void)
return in_task() ? ¤t->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
}
+/* Rules for generic atomic accesses. Called from fast-path. */
static __always_inline bool
is_atomic(const volatile void *ptr, size_t size, int type)
{
struct kcsan_ctx *ctx;
- if ((type & KCSAN_ACCESS_ATOMIC) != 0)
+ if (type & KCSAN_ACCESS_ATOMIC)
return true;
/*
@@ -201,16 +202,16 @@ is_atomic(const volatile void *ptr, size_t size, int type)
* as atomic. This allows using them also in atomic regions, such as
* seqlocks, without implicitly changing their semantics.
*/
- if ((type & KCSAN_ACCESS_ASSERT) != 0)
+ if (type & KCSAN_ACCESS_ASSERT)
return false;
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&
- (type & KCSAN_ACCESS_WRITE) != 0 && size <= sizeof(long) &&
+ (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&
IS_ALIGNED((unsigned long)ptr, size))
return true; /* Assume aligned writes up to word size are atomic. */
ctx = get_ctx();
- if (unlikely(ctx->atomic_next > 0)) {
+ if (ctx->atomic_next > 0) {
/*
* Because we do not have separate contexts for nested
* interrupts, in case atomic_next is set, we simply assume that
@@ -224,10 +225,8 @@ is_atomic(const volatile void *ptr, size_t size, int type)
--ctx->atomic_next; /* in task, or outer interrupt */
return true;
}
- if (unlikely(ctx->atomic_nest_count > 0 || ctx->in_flat_atomic))
- return true;
- return kcsan_is_atomic(ptr);
+ return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;
}
static __always_inline bool
@@ -367,6 +366,15 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
if (!kcsan_is_enabled())
goto out;
+ /*
+ * Special atomic rules: unlikely to be true, so we check them here in
+ * the slow-path, and not in the fast-path in is_atomic(). Call after
+ * kcsan_is_enabled(), as we may access memory that is not yet
+ * initialized during early boot.
+ */
+ if (!is_assert && kcsan_is_atomic_special(ptr))
+ goto out;
+
if (!check_encodable((unsigned long)ptr, size)) {
kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
goto out;
diff --git a/kernel/kcsan/debugfs.c b/kernel/kcsan/debugfs.c
index 2ff1961..72ee188 100644
--- a/kernel/kcsan/debugfs.c
+++ b/kernel/kcsan/debugfs.c
@@ -74,25 +74,34 @@ void kcsan_counter_dec(enum kcsan_counter_id id)
*/
static noinline void microbenchmark(unsigned long iters)
{
+ const struct kcsan_ctx ctx_save = current->kcsan_ctx;
+ const bool was_enabled = READ_ONCE(kcsan_enabled);
cycles_t cycles;
+ /* We may have been called from an atomic region; reset context. */
+ memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx));
+ /*
+ * Disable to benchmark fast-path for all accesses, and (expected
+ * negligible) call into slow-path, but never set up watchpoints.
+ */
+ WRITE_ONCE(kcsan_enabled, false);
+
pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
cycles = get_cycles();
while (iters--) {
- /*
- * We can run this benchmark from multiple tasks; this address
- * calculation increases likelyhood of some accesses
- * overlapping. Make the access type an atomic read, to never
- * set up watchpoints and test the fast-path only.
- */
- unsigned long addr =
- iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE);
- __kcsan_check_access((void *)addr, sizeof(long), KCSAN_ACCESS_ATOMIC);
+ unsigned long addr = iters & ((PAGE_SIZE << 8) - 1);
+ int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC :
+ (!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0);
+ __kcsan_check_access((void *)addr, sizeof(long), type);
}
cycles = get_cycles() - cycles;
pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles);
+
+ WRITE_ONCE(kcsan_enabled, was_enabled);
+ /* restore context */
+ current->kcsan_ctx = ctx_save;
}
/*
--
2.9.5
next prev parent reply other threads:[~2020-04-15 19:18 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-04-15 18:33 [PATCH kcsan 0/15] KCSAN updates for v5.8 Paul E. McKenney
2020-04-15 18:33 ` [PATCH v4 tip/core/rcu 01/15] kcsan: Add option to allow watcher interruptions paulmck
2020-04-15 18:33 ` [PATCH v4 tip/core/rcu 02/15] kcsan: Add option for verbose reporting paulmck
2020-04-15 18:33 ` paulmck [this message]
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 04/15] kcsan: Fix a typo in a comment paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 05/15] kcsan: Update Documentation/dev-tools/kcsan.rst paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 06/15] kcsan: Update API documentation in kcsan-checks.h paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 07/15] kcsan: Introduce report access_info and other_info paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 08/15] kcsan: Avoid blocking producers in prepare_report() paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 09/15] kcsan: Add support for scoped accesses paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 10/15] objtool, kcsan: Add explicit check functions to uaccess whitelist paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 11/15] kcsan: Introduce scoped ASSERT_EXCLUSIVE macros paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 12/15] kcsan: Move kcsan_{disable,enable}_current() to kcsan-checks.h paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 13/15] kcsan: Change data_race() to no longer require marking racing accesses paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 14/15] kcsan: Fix function matching in report paulmck
2020-04-15 18:34 ` [PATCH v4 tip/core/rcu 15/15] kcsan: Make reporting aware of KCSAN tests paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 01/10] tools/memory-model: Add recent references paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 02/10] tools/memory-model: Fix "conflict" definition paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 03/10] Documentation: LKMM: Move MP+onceassign+derefonce to new litmus-tests/rcu/ paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 04/10] Documentation: LKMM: Add litmus test for RCU GP guarantee where updater frees object paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 05/10] Documentation: LKMM: Add litmus test for RCU GP guarantee where reader stores paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 06/10] MAINTAINERS: Update maintainers for new Documentaion/litmus-tests/ paulmck
2020-04-15 21:39 ` Joe Perches
2020-04-16 0:17 ` Paul E. McKenney
2020-04-16 1:46 ` Joe Perches
2020-04-16 10:07 ` Paul E. McKenney
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 07/10] tools/memory-model: Add an exception for limitations on _unless() family paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 08/10] Documentation/litmus-tests: Introduce atomic directory paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 09/10] Documentation/litmus-tests/atomic: Add a test for atomic_set() paulmck
2020-04-15 18:49 ` [PATCH lkmm tip/core/rcu 10/10] Documentation/litmus-tests/atomic: Add a test for smp_mb__after_atomic() paulmck
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200415183411.12368-3-paulmck@kernel.org \
--to=paulmck@kernel.org \
--cc=andreyknvl@google.com \
--cc=boqun.feng@gmail.com \
--cc=cai@lca.pw \
--cc=dvyukov@google.com \
--cc=elver@google.com \
--cc=glider@google.com \
--cc=kasan-dev@googlegroups.com \
--cc=kernel-team@fb.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).