From: Matthew Wilcox <mawilcox@linuxonhyperv.com>
To: linux-kernel@vger.kernel.org,
Andrew Morton <akpm@linux-foundation.org>,
Konstantin Khlebnikov <koct9i@gmail.com>,
Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Matthew Wilcox <mawilcox@microsoft.com>,
linux-mm@kvack.org, linux-fsdevel@vger.kernel.org,
"Kirill A . Shutemov" <kirill.shutemov@linux.intel.com>
Subject: [PATCH v3 26/33] radix-tree: Fix replacement for multiorder entries
Date: Mon, 28 Nov 2016 13:50:30 -0800 [thread overview]
Message-ID: <1480369871-5271-27-git-send-email-mawilcox@linuxonhyperv.com> (raw)
In-Reply-To: <1480369871-5271-1-git-send-email-mawilcox@linuxonhyperv.com>
From: Matthew Wilcox <mawilcox@microsoft.com>
When replacing an entry with NULL, we need to delete any sibling entries.
Also account deleting exceptional entries properly.
Also fix a bug with radix_tree_iter_replace() where we would fail to
remove entirely freed nodes.
Also fix accounting bug when switching between normal and exceptional
entries with replace_slot.
Also add testcases for all these bugs.
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
---
lib/radix-tree.c | 60 +++++++++++++++++-------
tools/testing/radix-tree/multiorder.c | 87 ++++++++++++++++++++++++++++++-----
2 files changed, 119 insertions(+), 28 deletions(-)
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 9d24bec..a227727 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -976,6 +976,24 @@ void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index)
}
EXPORT_SYMBOL(radix_tree_lookup);
+static inline int slot_count(struct radix_tree_node *node,
+ void **slot)
+{
+ int n = 1;
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+ void *ptr = node_to_entry(slot);
+ unsigned offset = get_slot_offset(node, slot);
+ int i;
+
+ for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
+ if (node->slots[offset + i] != ptr)
+ break;
+ n++;
+ }
+#endif
+ return n;
+}
+
static void replace_slot(struct radix_tree_root *root,
struct radix_tree_node *node,
void **slot, void *item,
@@ -994,12 +1012,35 @@ static void replace_slot(struct radix_tree_root *root,
if (node) {
node->count += count;
- node->exceptional += exceptional;
+ if (exceptional) {
+ exceptional *= slot_count(node, slot);
+ node->exceptional += exceptional;
+ }
}
rcu_assign_pointer(*slot, item);
}
+static inline void delete_sibling_entries(struct radix_tree_node *node,
+ void **slot)
+{
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+ bool exceptional = radix_tree_exceptional_entry(*slot);
+ void *ptr = node_to_entry(slot);
+ unsigned offset = get_slot_offset(node, slot);
+ int i;
+
+ for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
+ if (node->slots[offset + i] != ptr)
+ break;
+ node->slots[offset + i] = NULL;
+ node->count--;
+ if (exceptional)
+ node->exceptional--;
+ }
+#endif
+}
+
/**
* __radix_tree_replace - replace item in a slot
* @root: radix tree root
@@ -1017,6 +1058,8 @@ void __radix_tree_replace(struct radix_tree_root *root,
void **slot, void *item,
radix_tree_update_node_t update_node, void *private)
{
+ if (!item)
+ delete_sibling_entries(node, slot);
/*
* This function supports replacing exceptional entries and
* deleting entries, but that needs accounting against the
@@ -1793,20 +1836,6 @@ void __radix_tree_delete_node(struct radix_tree_root *root,
delete_node(root, node, NULL, NULL);
}
-static inline void delete_sibling_entries(struct radix_tree_node *node,
- void *ptr, unsigned offset)
-{
-#ifdef CONFIG_RADIX_TREE_MULTIORDER
- int i;
- for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
- if (node->slots[offset + i] != ptr)
- break;
- node->slots[offset + i] = NULL;
- node->count--;
- }
-#endif
-}
-
/**
* radix_tree_delete_item - delete an item from a radix tree
* @root: radix tree root
@@ -1846,7 +1875,6 @@ void *radix_tree_delete_item(struct radix_tree_root *root,
for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
node_tag_clear(root, node, tag, offset);
- delete_sibling_entries(node, node_to_entry(slot), offset);
__radix_tree_replace(root, node, slot, NULL, NULL, NULL);
return entry;
diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c
index 5421f01..9757b89 100644
--- a/tools/testing/radix-tree/multiorder.c
+++ b/tools/testing/radix-tree/multiorder.c
@@ -410,8 +410,6 @@ static void __multiorder_split(int old_order, int new_order)
RADIX_TREE(tree, GFP_ATOMIC);
void **slot;
struct radix_tree_iter iter;
- struct radix_tree_node *node;
- void *item;
unsigned alloc;
radix_tree_preload(GFP_KERNEL);
@@ -434,58 +432,122 @@ static void __multiorder_split(int old_order, int new_order)
radix_tree_preload_end();
item_kill_tree(&tree);
+}
+
+static void __multiorder_split2(int old_order, int new_order)
+{
+ RADIX_TREE(tree, GFP_KERNEL);
+ void **slot;
+ struct radix_tree_iter iter;
+ struct radix_tree_node *node;
+ void *item;
- radix_tree_preload(GFP_KERNEL);
__radix_tree_insert(&tree, 0, old_order, (void *)0x12);
- radix_tree_preload_end();
item = __radix_tree_lookup(&tree, 0, &node, NULL);
assert(item == (void *)0x12);
assert(node->exceptional > 0);
- radix_tree_split_preload(old_order, new_order, GFP_KERNEL);
radix_tree_split(&tree, 0, new_order);
radix_tree_for_each_slot(slot, &tree, &iter, 0) {
radix_tree_iter_replace(&tree, &iter, slot,
item_create(iter.index, new_order));
}
- radix_tree_preload_end();
item = __radix_tree_lookup(&tree, 0, &node, NULL);
assert(item != (void *)0x12);
assert(node->exceptional == 0);
item_kill_tree(&tree);
+}
+
+static void __multiorder_split3(int old_order, int new_order)
+{
+ RADIX_TREE(tree, GFP_KERNEL);
+ void **slot;
+ struct radix_tree_iter iter;
+ struct radix_tree_node *node;
+ void *item;
- radix_tree_preload(GFP_KERNEL);
__radix_tree_insert(&tree, 0, old_order, (void *)0x12);
- radix_tree_preload_end();
item = __radix_tree_lookup(&tree, 0, &node, NULL);
assert(item == (void *)0x12);
assert(node->exceptional > 0);
- radix_tree_split_preload(old_order, new_order, GFP_KERNEL);
radix_tree_split(&tree, 0, new_order);
radix_tree_for_each_slot(slot, &tree, &iter, 0) {
radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16);
}
- radix_tree_preload_end();
item = __radix_tree_lookup(&tree, 0, &node, NULL);
assert(item == (void *)0x16);
assert(node->exceptional > 0);
item_kill_tree(&tree);
+
+ __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
+
+ item = __radix_tree_lookup(&tree, 0, &node, NULL);
+ assert(item == (void *)0x12);
+ assert(node->exceptional > 0);
+
+ radix_tree_split(&tree, 0, new_order);
+ radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+ if (iter.index == (1 << new_order))
+ radix_tree_iter_replace(&tree, &iter, slot,
+ (void *)0x16);
+ else
+ radix_tree_iter_replace(&tree, &iter, slot, NULL);
+ }
+
+ item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL);
+ assert(item == (void *)0x16);
+ assert(node->count == node->exceptional);
+ do {
+ node = node->parent;
+ if (!node)
+ break;
+ assert(node->count == 1);
+ assert(node->exceptional == 0);
+ } while (1);
+
+ item_kill_tree(&tree);
}
static void multiorder_split(void)
{
int i, j;
- for (i = 9; i < 19; i++)
- for (j = 0; j < i; j++)
+ for (i = 3; i < 11; i++)
+ for (j = 0; j < i; j++) {
__multiorder_split(i, j);
+ __multiorder_split2(i, j);
+ __multiorder_split3(i, j);
+ }
+}
+
+static void multiorder_account(void)
+{
+ RADIX_TREE(tree, GFP_KERNEL);
+ struct radix_tree_node *node;
+ void **slot;
+
+ item_insert_order(&tree, 0, 5);
+
+ __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
+ __radix_tree_lookup(&tree, 0, &node, NULL);
+ assert(node->count == node->exceptional * 2);
+ radix_tree_delete(&tree, 1 << 5);
+ assert(node->exceptional == 0);
+
+ __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
+ __radix_tree_lookup(&tree, 1 << 5, &node, &slot);
+ assert(node->count == node->exceptional * 2);
+ __radix_tree_replace(&tree, node, slot, NULL, NULL, NULL);
+ assert(node->exceptional == 0);
+
+ item_kill_tree(&tree);
}
void multiorder_checks(void)
@@ -507,6 +569,7 @@ void multiorder_checks(void)
multiorder_tagged_iteration();
multiorder_join();
multiorder_split();
+ multiorder_account();
radix_tree_cpu_dead(0);
}
--
2.10.2
next prev parent reply other threads:[~2016-11-28 20:05 UTC|newest]
Thread overview: 74+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-11-28 21:50 [PATCH v3 00/33] Radix tree patches for 4.10 Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 01/33] radix tree test suite: Fix compilation Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 02/33] tools: Add WARN_ON_ONCE Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 03/33] radix tree test suite: Allow GFP_ATOMIC allocations to fail Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 04/33] radix tree test suite: Track preempt_count Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 05/33] radix tree test suite: Free preallocated nodes Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 06/33] radix tree test suite: Make runs more reproducible Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 07/33] radix tree test suite: iteration test misuses RCU Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 08/33] radix tree test suite: benchmark for iterator Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 09/33] radix tree test suite: Use rcu_barrier Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 10/33] radix tree test suite: Handle exceptional entries Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 11/33] radix tree test suite: record order in each item Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 12/33] tools: Add more bitmap functions Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 13/33] radix tree test suite: Use common find-bit code Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 14/33] radix-tree: Fix typo Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 15/33] radix-tree: Move rcu_head into a union with private_list Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 16/33] radix-tree: Create node_tag_set() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 17/33] radix-tree: Make radix_tree_find_next_bit more useful Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 18/33] radix-tree: Improve dump output Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 19/33] btrfs: Fix race in btrfs_free_dummy_fs_info() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 20/33] radix-tree: Improve multiorder iterators Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 21/33] radix-tree: Delete radix_tree_locate_item() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 22/33] radix-tree: Delete radix_tree_range_tag_if_tagged() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 23/33] radix-tree: Add radix_tree_join Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 24/33] radix-tree: Add radix_tree_split Matthew Wilcox
2016-11-29 0:02 ` Randy Dunlap
2016-11-29 14:26 ` Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 25/33] radix-tree: Add radix_tree_split_preload() Matthew Wilcox
2016-11-28 21:50 ` Matthew Wilcox [this message]
2016-11-28 21:50 ` [PATCH v3 27/33] radix tree test suite: Check multiorder iteration Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 28/33] idr: Add ida_is_empty Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 29/33] tpm: Use idr_find(), not idr_find_slowpath() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 30/33] rxrpc: Abstract away knowledge of IDR internals Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 31/33] idr: Reduce the number of bits per level from 8 to 6 Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 32/33] radix tree test suite: Add some more functionality Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 33/33] Reimplement IDR and IDA using the radix tree Matthew Wilcox
2016-12-06 20:44 ` Andrew Morton
2016-12-06 21:17 ` Matthew Wilcox
2016-12-06 21:25 ` Andrew Morton
2016-11-28 21:50 ` [PATCH v3 00/33] Radix tree patches for 4.10 Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 01/33] radix tree test suite: Fix compilation Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 02/33] tools: Add WARN_ON_ONCE Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 03/33] radix tree test suite: Allow GFP_ATOMIC allocations to fail Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 04/33] radix tree test suite: Track preempt_count Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 05/33] radix tree test suite: Free preallocated nodes Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 06/33] radix tree test suite: Make runs more reproducible Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 07/33] radix tree test suite: iteration test misuses RCU Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 08/33] radix tree test suite: benchmark for iterator Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 09/33] radix tree test suite: Use rcu_barrier Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 10/33] radix tree test suite: Handle exceptional entries Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 11/33] radix tree test suite: record order in each item Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 12/33] tools: Add more bitmap functions Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 13/33] radix tree test suite: Use common find-bit code Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 14/33] radix-tree: Fix typo Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 15/33] radix-tree: Move rcu_head into a union with private_list Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 16/33] radix-tree: Create node_tag_set() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 17/33] radix-tree: Make radix_tree_find_next_bit more useful Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 18/33] radix-tree: Improve dump output Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 19/33] btrfs: Fix race in btrfs_free_dummy_fs_info() Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 20/33] radix-tree: Improve multiorder iterators Matthew Wilcox
2016-11-28 21:50 ` [PATCH v3 21/33] radix-tree: Delete radix_tree_locate_item() Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 22/33] radix-tree: Delete radix_tree_range_tag_if_tagged() Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 23/33] radix-tree: Add radix_tree_join Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 24/33] radix-tree: Add radix_tree_split Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 25/33] radix-tree: Add radix_tree_split_preload() Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 26/33] radix-tree: Fix replacement for multiorder entries Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 27/33] radix tree test suite: Check multiorder iteration Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 28/33] idr: Add ida_is_empty Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 29/33] tpm: Use idr_find(), not idr_find_slowpath() Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 30/33] rxrpc: Abstract away knowledge of IDR internals Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 31/33] idr: Reduce the number of bits per level from 8 to 6 Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 32/33] radix tree test suite: Add some more functionality Matthew Wilcox
2016-11-28 21:51 ` [PATCH v3 33/33] Reimplement IDR and IDA using the radix tree Matthew Wilcox
2016-11-29 11:24 ` [PATCH v3 00/33] Radix tree patches for 4.10 Kirill A. Shutemov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1480369871-5271-27-git-send-email-mawilcox@linuxonhyperv.com \
--to=mawilcox@linuxonhyperv.com \
--cc=akpm@linux-foundation.org \
--cc=kirill.shutemov@linux.intel.com \
--cc=koct9i@gmail.com \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mawilcox@microsoft.com \
--cc=ross.zwisler@linux.intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).