From: Will Deacon <will@kernel.org>
To: linux-kernel@vger.kernel.org
Cc: Will Deacon <will@kernel.org>, Eric Dumazet <edumazet@google.com>,
Jann Horn <jannh@google.com>, Kees Cook <keescook@chromium.org>,
Maddie Stone <maddiestone@google.com>,
Marco Elver <elver@google.com>,
"Paul E . McKenney" <paulmck@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Thomas Gleixner <tglx@linutronix.de>,
kernel-team@android.com, kernel-hardening@lists.openwall.com
Subject: [RFC PATCH 11/21] list: Add integrity checking to hlist implementation
Date: Tue, 24 Mar 2020 15:36:33 +0000 [thread overview]
Message-ID: <20200324153643.15527-12-will@kernel.org> (raw)
In-Reply-To: <20200324153643.15527-1-will@kernel.org>
Extend the 'hlist' implementation so that it can optionally perform
integrity checking in a similar fashion to the standard 'list' code
when CONFIG_CHECK_INTEGRITY_LIST=y.
Cc: Kees Cook <keescook@chromium.org>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Will Deacon <will@kernel.org>
---
include/linux/list.h | 41 ++++++++++++++++++++-
include/linux/rculist.h | 80 ++++++++++++++++++++++-------------------
lib/list_debug.c | 79 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 162 insertions(+), 38 deletions(-)
diff --git a/include/linux/list.h b/include/linux/list.h
index 2bef081afa69..96ede36a5614 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -41,6 +41,13 @@ extern bool __list_add_valid(struct list_head *new,
struct list_head *prev,
struct list_head *next);
extern bool __list_del_entry_valid(struct list_head *entry);
+extern bool __hlist_add_before_valid(struct hlist_node *new,
+ struct hlist_node *next);
+extern bool __hlist_add_behind_valid(struct hlist_node *new,
+ struct hlist_node *prev);
+extern bool __hlist_add_head_valid(struct hlist_node *new,
+ struct hlist_head *head);
+extern bool __hlist_del_valid(struct hlist_node *node);
#else
static inline bool __list_add_valid(struct list_head *new,
struct list_head *prev,
@@ -52,6 +59,25 @@ static inline bool __list_del_entry_valid(struct list_head *entry)
{
return true;
}
+static inline bool __hlist_add_before_valid(struct hlist_node *new,
+ struct hlist_node *next)
+{
+ return true;
+}
+static inline bool __hlist_add_behind_valid(struct hlist_node *new,
+ struct hlist_node *prev)
+{
+ return true;
+}
+static inline bool __hlist_add_head_valid(struct hlist_node *new,
+ struct hlist_head *head)
+{
+ return true;
+}
+static inline bool __hlist_del_valid(struct hlist_node *node)
+{
+ return true;
+}
#endif
/*
@@ -796,6 +822,9 @@ static inline void __hlist_del(struct hlist_node *n)
struct hlist_node *next = n->next;
struct hlist_node **pprev = n->pprev;
+ if (!__hlist_del_valid(n))
+ return;
+
WRITE_ONCE(*pprev, next);
if (next)
WRITE_ONCE(next->pprev, pprev);
@@ -840,6 +869,10 @@ static inline void hlist_del_init(struct hlist_node *n)
static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
{
struct hlist_node *first = h->first;
+
+ if (!__hlist_add_head_valid(n, h))
+ return;
+
n->next = first;
if (first)
first->pprev = &n->next;
@@ -855,6 +888,9 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
static inline void hlist_add_before(struct hlist_node *n,
struct hlist_node *next)
{
+ if (!__hlist_add_before_valid(n, next))
+ return;
+
n->pprev = next->pprev;
n->next = next;
next->pprev = &n->next;
@@ -862,13 +898,16 @@ static inline void hlist_add_before(struct hlist_node *n,
}
/**
- * hlist_add_behing - add a new entry after the one specified
+ * hlist_add_behind - add a new entry after the one specified
* @n: new entry to be added
* @prev: hlist node to add it after, which must be non-NULL
*/
static inline void hlist_add_behind(struct hlist_node *n,
struct hlist_node *prev)
{
+ if (!__hlist_add_behind_valid(n, prev))
+ return;
+
n->next = prev->next;
prev->next = n;
n->pprev = &prev->next;
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index 9f313e4999fe..6f3eb7758fd8 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -537,6 +537,9 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,
{
struct hlist_node *first = h->first;
+ if (!__hlist_add_head_valid(n, h))
+ return;
+
n->next = first;
WRITE_ONCE(n->pprev, &h->first);
rcu_assign_pointer(hlist_first_rcu(h), n);
@@ -544,43 +547,6 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,
WRITE_ONCE(first->pprev, &n->next);
}
-/**
- * hlist_add_tail_rcu
- * @n: the element to add to the hash list.
- * @h: the list to add to.
- *
- * Description:
- * Adds the specified element to the specified hlist,
- * while permitting racing traversals.
- *
- * The caller must take whatever precautions are necessary
- * (such as holding appropriate locks) to avoid racing
- * with another list-mutation primitive, such as hlist_add_head_rcu()
- * or hlist_del_rcu(), running on this same list.
- * However, it is perfectly legal to run concurrently with
- * the _rcu list-traversal primitives, such as
- * hlist_for_each_entry_rcu(), used to prevent memory-consistency
- * problems on Alpha CPUs. Regardless of the type of CPU, the
- * list-traversal primitive must be guarded by rcu_read_lock().
- */
-static inline void hlist_add_tail_rcu(struct hlist_node *n,
- struct hlist_head *h)
-{
- struct hlist_node *i, *last = NULL;
-
- /* Note: write side code, so rcu accessors are not needed. */
- for (i = h->first; i; i = i->next)
- last = i;
-
- if (last) {
- n->next = last->next;
- WRITE_ONCE(n->pprev, &last->next);
- rcu_assign_pointer(hlist_next_rcu(last), n);
- } else {
- hlist_add_head_rcu(n, h);
- }
-}
-
/**
* hlist_add_before_rcu
* @n: the new element to add to the hash list.
@@ -602,6 +568,9 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n,
static inline void hlist_add_before_rcu(struct hlist_node *n,
struct hlist_node *next)
{
+ if (!__hlist_add_before_valid(n, next))
+ return;
+
WRITE_ONCE(n->pprev, next->pprev);
n->next = next;
rcu_assign_pointer(hlist_pprev_rcu(n), n);
@@ -629,6 +598,9 @@ static inline void hlist_add_before_rcu(struct hlist_node *n,
static inline void hlist_add_behind_rcu(struct hlist_node *n,
struct hlist_node *prev)
{
+ if (!__hlist_add_behind_valid(n, prev))
+ return;
+
n->next = prev->next;
WRITE_ONCE(n->pprev, &prev->next);
rcu_assign_pointer(hlist_next_rcu(prev), n);
@@ -636,6 +608,40 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
WRITE_ONCE(n->next->pprev, &n->next);
}
+/**
+ * hlist_add_tail_rcu
+ * @n: the element to add to the hash list.
+ * @h: the list to add to.
+ *
+ * Description:
+ * Adds the specified element to the specified hlist,
+ * while permitting racing traversals.
+ *
+ * The caller must take whatever precautions are necessary
+ * (such as holding appropriate locks) to avoid racing
+ * with another list-mutation primitive, such as hlist_add_head_rcu()
+ * or hlist_del_rcu(), running on this same list.
+ * However, it is perfectly legal to run concurrently with
+ * the _rcu list-traversal primitives, such as
+ * hlist_for_each_entry_rcu(), used to prevent memory-consistency
+ * problems on Alpha CPUs. Regardless of the type of CPU, the
+ * list-traversal primitive must be guarded by rcu_read_lock().
+ */
+static inline void hlist_add_tail_rcu(struct hlist_node *n,
+ struct hlist_head *h)
+{
+ struct hlist_node *i, *last = NULL;
+
+ /* Note: write side code, so rcu accessors are not needed. */
+ for (i = h->first; i; i = i->next)
+ last = i;
+
+ if (last)
+ hlist_add_behind_rcu(n, last);
+ else
+ hlist_add_head_rcu(n, h);
+}
+
#define __hlist_for_each_rcu(pos, head) \
for (pos = rcu_dereference(hlist_first_rcu(head)); \
pos; \
diff --git a/lib/list_debug.c b/lib/list_debug.c
index 57bf685af2ef..03234ebd18c9 100644
--- a/lib/list_debug.c
+++ b/lib/list_debug.c
@@ -60,3 +60,82 @@ bool __list_del_entry_valid(struct list_head *entry)
}
EXPORT_SYMBOL(__list_del_entry_valid);
+
+static bool __hlist_add_valid(struct hlist_node *new, struct hlist_node *prev,
+ struct hlist_node *next)
+{
+ if (CHECK_DATA_CORRUPTION(next && next->pprev != &prev->next,
+ "hlist_add corruption: next->pprev should be &prev->next (%px), but was %px (next=%px)\n",
+ &prev->next, next->pprev, next) ||
+ CHECK_DATA_CORRUPTION(prev->next != next,
+ "hlist_add corruption: prev->next should be next (%px), but was %px (prev=%px)\n",
+ next, prev->next, prev) ||
+ CHECK_DATA_CORRUPTION(new == prev || new == next,
+ "hlist_add double add: new=%px, prev=%px, next=%px\n",
+ new, prev, next))
+ return false;
+
+ return true;
+}
+
+bool __hlist_add_before_valid(struct hlist_node *new, struct hlist_node *next)
+{
+ struct hlist_node *prev;
+
+ prev = container_of(next->pprev, struct hlist_node, next);
+ return __hlist_add_valid(new, prev, next);
+}
+EXPORT_SYMBOL(__hlist_add_before_valid);
+
+bool __hlist_add_behind_valid(struct hlist_node *new, struct hlist_node *prev)
+{
+ return __hlist_add_valid(new, prev, prev->next);
+}
+EXPORT_SYMBOL(__hlist_add_behind_valid);
+
+bool __hlist_add_head_valid(struct hlist_node *new, struct hlist_head *head)
+{
+ struct hlist_node *first = head->first;
+
+ if (CHECK_DATA_CORRUPTION(first && first->pprev != &head->first,
+ "hlist_add_head corruption: first->pprev should be &head->first (%px), but was %px (first=%px)",
+ &head->first, first->pprev, first) ||
+ CHECK_DATA_CORRUPTION(new == first,
+ "hlist_add_head double add: new (%px) == first (%px)",
+ new, first))
+ return false;
+
+ return true;
+}
+EXPORT_SYMBOL(__hlist_add_head_valid);
+
+bool __hlist_del_valid(struct hlist_node *node)
+{
+ struct hlist_node *prev, *next = node->next;
+
+ if (CHECK_DATA_CORRUPTION(next == LIST_POISON1,
+ "hlist_del corruption: %px->next is LIST_POISON1 (%px)\n",
+ node, LIST_POISON1) ||
+ CHECK_DATA_CORRUPTION(node->pprev == LIST_POISON2,
+ "hlist_del corruption: %px->pprev is LIST_POISON2 (%px)\n",
+ node, LIST_POISON2))
+ return false;
+
+ /*
+ * If we want to validate the previous node's forward linkage,
+ * then we must be able to treat the head like a normal node.
+ */
+ BUILD_BUG_ON(offsetof(struct hlist_node, next) !=
+ offsetof(struct hlist_head, first));
+ prev = container_of(node->pprev, struct hlist_node, next);
+ if (CHECK_DATA_CORRUPTION(prev->next != node,
+ "hlist_del corruption: prev->next should be %px, but was %px\n",
+ node, prev->next) ||
+ CHECK_DATA_CORRUPTION(next && next->pprev != &node->next,
+ "hlist_del corruption: next->pprev should be %px, but was %px\n",
+ &node->next, next->pprev))
+ return false;
+
+ return true;
+}
+EXPORT_SYMBOL(__hlist_del_valid);
--
2.20.1
next prev parent reply other threads:[~2020-03-24 15:37 UTC|newest]
Thread overview: 58+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-24 15:36 [RFC PATCH 00/21] Improve list integrity checking Will Deacon
2020-03-24 15:36 ` [RFC PATCH 01/21] list: Remove hlist_unhashed_lockless() Will Deacon
2020-03-24 16:27 ` Greg KH
2020-03-30 23:05 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 02/21] list: Remove hlist_nulls_unhashed_lockless() Will Deacon
2020-03-24 16:27 ` Greg KH
2020-03-30 23:07 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 03/21] list: Annotate lockless list primitives with data_race() Will Deacon
2020-03-24 16:20 ` Jann Horn
2020-03-24 16:26 ` Greg KH
2020-03-24 16:38 ` Jann Horn
2020-03-24 16:59 ` Greg KH
2020-03-24 18:22 ` Jann Horn
2020-03-24 16:23 ` Marco Elver
2020-03-24 21:33 ` Will Deacon
2020-03-31 13:10 ` Will Deacon
2020-04-01 6:34 ` Marco Elver
2020-04-01 8:40 ` Will Deacon
2020-05-08 13:46 ` [tip: locking/kcsan] kcsan: Change data_race() to no longer require marking racing accesses tip-bot2 for Marco Elver
2020-03-24 16:51 ` [RFC PATCH 03/21] list: Annotate lockless list primitives with data_race() Peter Zijlstra
2020-03-24 16:56 ` Jann Horn
2020-03-24 21:32 ` Will Deacon
2020-03-30 23:13 ` Paul E. McKenney
2020-04-24 17:39 ` Will Deacon
2020-04-27 19:24 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 04/21] timers: Use hlist_unhashed() instead of open-coding in timer_pending() Will Deacon
2020-03-24 16:30 ` Greg KH
2020-03-24 15:36 ` [RFC PATCH 05/21] list: Comment missing WRITE_ONCE() in __list_del() Will Deacon
2020-03-30 23:14 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 06/21] list: Remove superfluous WRITE_ONCE() from hlist_nulls implementation Will Deacon
2020-03-30 23:21 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 07/21] Revert "list: Use WRITE_ONCE() when adding to lists and hlists" Will Deacon
2020-03-30 23:19 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 08/21] Revert "list: Use WRITE_ONCE() when initializing list_head structures" Will Deacon
2020-03-30 23:25 ` Paul E. McKenney
2020-03-31 13:11 ` Will Deacon
2020-03-31 13:47 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 09/21] list: Remove unnecessary WRITE_ONCE() from hlist_bl_add_before() Will Deacon
2020-03-30 23:30 ` Paul E. McKenney
2020-03-31 12:37 ` Will Deacon
2020-03-24 15:36 ` [RFC PATCH 10/21] kernel-hacking: Make DEBUG_{LIST,PLIST,SG,NOTIFIERS} non-debug options Will Deacon
2020-03-24 16:42 ` Greg KH
2020-03-24 15:36 ` Will Deacon [this message]
2020-03-24 15:36 ` [RFC PATCH 12/21] list: Poison ->next pointer for non-RCU deletion of 'hlist_nulls_node' Will Deacon
2020-03-30 23:32 ` Paul E. McKenney
2020-03-24 15:36 ` [RFC PATCH 13/21] list: Add integrity checking to hlist_nulls implementation Will Deacon
2020-03-24 15:36 ` [RFC PATCH 14/21] plist: Use CHECK_DATA_CORRUPTION instead of explicit {BUG,WARN}_ON() Will Deacon
2020-03-24 16:42 ` Greg KH
2020-03-24 15:36 ` [RFC PATCH 15/21] list_bl: Use CHECK_DATA_CORRUPTION instead of custom BUG_ON() wrapper Will Deacon
2020-03-24 15:36 ` [RFC PATCH 16/21] list_bl: Extend integrity checking in deletion routines Will Deacon
2020-03-24 15:36 ` [RFC PATCH 17/21] linux/bit_spinlock.h: Include linux/processor.h Will Deacon
2020-03-24 16:28 ` Greg KH
2020-03-24 21:08 ` Will Deacon
2020-03-24 15:36 ` [RFC PATCH 18/21] list_bl: Move integrity checking out of line Will Deacon
2020-03-24 15:36 ` [RFC PATCH 19/21] list_bl: Extend integrity checking to cover the same cases as 'hlist' Will Deacon
2020-03-24 15:36 ` [RFC PATCH 20/21] list: Format CHECK_DATA_CORRUPTION error messages consistently Will Deacon
2020-03-24 16:40 ` Greg KH
2020-03-24 15:36 ` [RFC PATCH 21/21] lkdtm: Extend list corruption checks Will Deacon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200324153643.15527-12-will@kernel.org \
--to=will@kernel.org \
--cc=edumazet@google.com \
--cc=elver@google.com \
--cc=jannh@google.com \
--cc=keescook@chromium.org \
--cc=kernel-hardening@lists.openwall.com \
--cc=kernel-team@android.com \
--cc=linux-kernel@vger.kernel.org \
--cc=maddiestone@google.com \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).