From: Will Deacon <will@kernel.org>
To: linux-kernel@vger.kernel.org
Cc: Will Deacon <will@kernel.org>, Kees Cook <keescook@chromium.org>,
Ingo Molnar <mingo@kernel.org>,
Elena Reshetova <elena.reshetova@intel.com>,
Peter Zijlstra <peterz@infradead.org>,
Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Hanjun Guo <guohanjun@huawei.com>,
Jan Glauber <jglauber@marvell.com>
Subject: [PATCH v2 5/6] lib/refcount: Improve performance of generic REFCOUNT_FULL code
Date: Tue, 27 Aug 2019 17:32:03 +0100 [thread overview]
Message-ID: <20190827163204.29903-6-will@kernel.org> (raw)
In-Reply-To: <20190827163204.29903-1-will@kernel.org>
Rewrite the generic REFCOUNT_FULL implementation so that the saturation
point is moved to INT_MIN / 2. This allows us to defer the sanity checks
until after the atomic operation, which removes many uses of cmpxchg()
in favour of atomic_fetch_{add,sub}().
Cc: Kees Cook <keescook@chromium.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Elena Reshetova <elena.reshetova@intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Jan Glauber <jglauber@marvell.com>
Signed-off-by: Will Deacon <will@kernel.org>
---
include/linux/refcount.h | 87 +++++++++++++++++++-----------------------------
1 file changed, 34 insertions(+), 53 deletions(-)
diff --git a/include/linux/refcount.h b/include/linux/refcount.h
index e719b5b1220e..7f9aa6511142 100644
--- a/include/linux/refcount.h
+++ b/include/linux/refcount.h
@@ -47,8 +47,8 @@ static inline unsigned int refcount_read(const refcount_t *r)
#ifdef CONFIG_REFCOUNT_FULL
#include <linux/bug.h>
-#define REFCOUNT_MAX (UINT_MAX - 1)
-#define REFCOUNT_SATURATED UINT_MAX
+#define REFCOUNT_MAX INT_MAX
+#define REFCOUNT_SATURATED (INT_MIN / 2)
/*
* Variant of atomic_t specialized for reference counts.
@@ -109,25 +109,19 @@ static inline unsigned int refcount_read(const refcount_t *r)
*/
static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
{
- unsigned int new, val = atomic_read(&r->refs);
+ int old = refcount_read(r);
do {
- if (!val)
- return false;
-
- if (unlikely(val == REFCOUNT_SATURATED))
- return true;
-
- new = val + i;
- if (new < val)
- new = REFCOUNT_SATURATED;
+ if (!old)
+ break;
+ } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));
- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
-
- WARN_ONCE(new == REFCOUNT_SATURATED,
- "refcount_t: saturated; leaking memory.\n");
+ if (unlikely(old < 0 || old + i < 0)) {
+ refcount_set(r, REFCOUNT_SATURATED);
+ WARN_ONCE(1, "refcount_t: saturated; leaking memory.\n");
+ }
- return true;
+ return old;
}
/**
@@ -148,7 +142,13 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
*/
static inline void refcount_add(int i, refcount_t *r)
{
- WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
+ int old = atomic_fetch_add_relaxed(i, &r->refs);
+
+ WARN_ONCE(!old, "refcount_t: addition on 0; use-after-free.\n");
+ if (unlikely(old <= 0 || old + i <= 0)) {
+ refcount_set(r, REFCOUNT_SATURATED);
+ WARN_ONCE(old, "refcount_t: saturated; leaking memory.\n");
+ }
}
/**
@@ -166,23 +166,7 @@ static inline void refcount_add(int i, refcount_t *r)
*/
static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
{
- unsigned int new, val = atomic_read(&r->refs);
-
- do {
- new = val + 1;
-
- if (!val)
- return false;
-
- if (unlikely(!new))
- return true;
-
- } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
-
- WARN_ONCE(new == REFCOUNT_SATURATED,
- "refcount_t: saturated; leaking memory.\n");
-
- return true;
+ return refcount_add_not_zero(1, r);
}
/**
@@ -199,7 +183,7 @@ static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
*/
static inline void refcount_inc(refcount_t *r)
{
- WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
+ refcount_add(1, r);
}
/**
@@ -224,26 +208,19 @@ static inline void refcount_inc(refcount_t *r)
*/
static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
{
- unsigned int new, val = atomic_read(&r->refs);
-
- do {
- if (unlikely(val == REFCOUNT_SATURATED))
- return false;
+ int old = atomic_fetch_sub_release(i, &r->refs);
- new = val - i;
- if (new > val) {
- WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
- return false;
- }
-
- } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
-
- if (!new) {
+ if (old == i) {
smp_acquire__after_ctrl_dep();
return true;
}
- return false;
+ if (unlikely(old - i < 0)) {
+ refcount_set(r, REFCOUNT_SATURATED);
+ WARN_ONCE(1, "refcount_t: underflow; use-after-free.\n");
+ }
+
+ return false;
}
/**
@@ -276,9 +253,13 @@ static inline __must_check bool refcount_dec_and_test(refcount_t *r)
*/
static inline void refcount_dec(refcount_t *r)
{
- WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
-}
+ int old = atomic_fetch_sub_release(1, &r->refs);
+ if (unlikely(old <= 1)) {
+ refcount_set(r, REFCOUNT_SATURATED);
+ WARN_ONCE(1, "refcount_t: decrement hit 0; leaking memory.\n");
+ }
+}
#else /* CONFIG_REFCOUNT_FULL */
#define REFCOUNT_MAX INT_MAX
--
2.11.0
next prev parent reply other threads:[~2019-08-27 16:32 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-08-27 16:31 [PATCH v2 0/6] Rework REFCOUNT_FULL using atomic_fetch_* operations Will Deacon
2019-08-27 16:31 ` [PATCH v2 1/6] lib/refcount: Define constants for saturation and max refcount values Will Deacon
2019-08-27 16:32 ` [PATCH v2 2/6] lib/refcount: Ensure integer operands are treated as signed Will Deacon
2019-08-27 16:32 ` [PATCH v2 3/6] lib/refcount: Remove unused refcount_*_checked() variants Will Deacon
2019-08-27 16:32 ` [PATCH v2 4/6] lib/refcount: Move bulk of REFCOUNT_FULL implementation into header Will Deacon
2019-08-27 16:32 ` Will Deacon [this message]
2019-08-27 16:32 ` [PATCH v2 6/6] lib/refcount: Consolidate REFCOUNT_{MAX,SATURATED} definitions Will Deacon
2019-08-27 17:51 ` [PATCH v2 0/6] Rework REFCOUNT_FULL using atomic_fetch_* operations Kees Cook
2019-08-28 7:30 ` Peter Zijlstra
2019-08-28 14:14 ` Will Deacon
2019-08-28 21:03 ` Kees Cook
2019-08-31 17:48 ` Ard Biesheuvel
2019-08-31 19:02 ` Kees Cook
2019-08-31 20:54 ` Ard Biesheuvel
2019-09-06 13:43 ` Will Deacon
2019-09-07 1:57 ` Hanjun Guo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190827163204.29903-6-will@kernel.org \
--to=will@kernel.org \
--cc=ard.biesheuvel@linaro.org \
--cc=elena.reshetova@intel.com \
--cc=guohanjun@huawei.com \
--cc=jglauber@marvell.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).