From: "tip-bot2 for Peter Zijlstra" <tip-bot2@linutronix.de>
To: linux-tip-commits@vger.kernel.org
Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org>,
Masami Hiramatsu <mhiramat@kernel.org>,
Ingo Molnar <mingo@kernel.org>, x86 <x86@kernel.org>,
LKML <linux-kernel@vger.kernel.org>
Subject: [tip: perf/kprobes] kprobes: Replace rp->free_instance with freelist
Date: Mon, 12 Oct 2020 17:08:24 -0000 [thread overview]
Message-ID: <160252250482.7002.18034659363554741944.tip-bot2@tip-bot2> (raw)
In-Reply-To: <159870623583.1229682.17472357584134058687.stgit@devnote2>
The following commit has been merged into the perf/kprobes branch of tip:
Commit-ID: 6e426e0fcd20ce144bb93e00b70df51e9f2e08c3
Gitweb: https://git.kernel.org/tip/6e426e0fcd20ce144bb93e00b70df51e9f2e08c3
Author: Peter Zijlstra <peterz@infradead.org>
AuthorDate: Sat, 29 Aug 2020 22:03:56 +09:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Mon, 12 Oct 2020 18:27:28 +02:00
kprobes: Replace rp->free_instance with freelist
Gets rid of rp->lock, and as a result kretprobes are now fully
lockless.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/159870623583.1229682.17472357584134058687.stgit@devnote2
---
include/linux/kprobes.h | 8 +++---
kernel/kprobes.c | 56 +++++++++++++++++-----------------------
2 files changed, 28 insertions(+), 36 deletions(-)
diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
index 00cf442..b7824e3 100644
--- a/include/linux/kprobes.h
+++ b/include/linux/kprobes.h
@@ -28,6 +28,7 @@
#include <linux/mutex.h>
#include <linux/ftrace.h>
#include <linux/refcount.h>
+#include <linux/freelist.h>
#include <asm/kprobes.h>
#ifdef CONFIG_KPROBES
@@ -157,17 +158,16 @@ struct kretprobe {
int maxactive;
int nmissed;
size_t data_size;
- struct hlist_head free_instances;
+ struct freelist_head freelist;
struct kretprobe_holder *rph;
- raw_spinlock_t lock;
};
struct kretprobe_instance {
union {
- struct llist_node llist;
- struct hlist_node hlist;
+ struct freelist_node freelist;
struct rcu_head rcu;
};
+ struct llist_node llist;
struct kretprobe_holder *rph;
kprobe_opcode_t *ret_addr;
void *fp;
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 850ee36..30b8fe7 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1228,11 +1228,8 @@ static void recycle_rp_inst(struct kretprobe_instance *ri)
{
struct kretprobe *rp = get_kretprobe(ri);
- INIT_HLIST_NODE(&ri->hlist);
if (likely(rp)) {
- raw_spin_lock(&rp->lock);
- hlist_add_head(&ri->hlist, &rp->free_instances);
- raw_spin_unlock(&rp->lock);
+ freelist_add(&ri->freelist, &rp->freelist);
} else
call_rcu(&ri->rcu, free_rp_inst_rcu);
}
@@ -1290,11 +1287,14 @@ NOKPROBE_SYMBOL(kprobe_flush_task);
static inline void free_rp_inst(struct kretprobe *rp)
{
struct kretprobe_instance *ri;
- struct hlist_node *next;
+ struct freelist_node *node;
int count = 0;
- hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
- hlist_del(&ri->hlist);
+ node = rp->freelist.head;
+ while (node) {
+ ri = container_of(node, struct kretprobe_instance, freelist);
+ node = node->next;
+
kfree(ri);
count++;
}
@@ -1925,32 +1925,26 @@ NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
{
struct kretprobe *rp = container_of(p, struct kretprobe, kp);
- unsigned long flags = 0;
struct kretprobe_instance *ri;
+ struct freelist_node *fn;
- /* TODO: consider to only swap the RA after the last pre_handler fired */
- raw_spin_lock_irqsave(&rp->lock, flags);
- if (!hlist_empty(&rp->free_instances)) {
- ri = hlist_entry(rp->free_instances.first,
- struct kretprobe_instance, hlist);
- hlist_del(&ri->hlist);
- raw_spin_unlock_irqrestore(&rp->lock, flags);
-
- if (rp->entry_handler && rp->entry_handler(ri, regs)) {
- raw_spin_lock_irqsave(&rp->lock, flags);
- hlist_add_head(&ri->hlist, &rp->free_instances);
- raw_spin_unlock_irqrestore(&rp->lock, flags);
- return 0;
- }
-
- arch_prepare_kretprobe(ri, regs);
+ fn = freelist_try_get(&rp->freelist);
+ if (!fn) {
+ rp->nmissed++;
+ return 0;
+ }
- __llist_add(&ri->llist, ¤t->kretprobe_instances);
+ ri = container_of(fn, struct kretprobe_instance, freelist);
- } else {
- rp->nmissed++;
- raw_spin_unlock_irqrestore(&rp->lock, flags);
+ if (rp->entry_handler && rp->entry_handler(ri, regs)) {
+ freelist_add(&ri->freelist, &rp->freelist);
+ return 0;
}
+
+ arch_prepare_kretprobe(ri, regs);
+
+ __llist_add(&ri->llist, ¤t->kretprobe_instances);
+
return 0;
}
NOKPROBE_SYMBOL(pre_handler_kretprobe);
@@ -2007,8 +2001,7 @@ int register_kretprobe(struct kretprobe *rp)
rp->maxactive = num_possible_cpus();
#endif
}
- raw_spin_lock_init(&rp->lock);
- INIT_HLIST_HEAD(&rp->free_instances);
+ rp->freelist.head = NULL;
rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
if (!rp->rph)
return -ENOMEM;
@@ -2023,8 +2016,7 @@ int register_kretprobe(struct kretprobe *rp)
return -ENOMEM;
}
inst->rph = rp->rph;
- INIT_HLIST_NODE(&inst->hlist);
- hlist_add_head(&inst->hlist, &rp->free_instances);
+ freelist_add(&inst->freelist, &rp->freelist);
}
refcount_set(&rp->rph->ref, i);
next prev parent reply other threads:[~2020-10-12 17:08 UTC|newest]
Thread overview: 73+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-08-29 12:59 [PATCH v5 00/21] kprobes: Unify kretprobe trampoline handlers and make kretprobe lockless Masami Hiramatsu
2020-08-29 13:00 ` [PATCH v5 01/21] kprobes: Add generic kretprobe trampoline handler Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:00 ` [PATCH v5 02/21] x86/kprobes: Use " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:00 ` [PATCH v5 03/21] arm: kprobes: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:00 ` [PATCH v5 04/21] arm64: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:00 ` [PATCH v5 05/21] arc: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:00 ` [PATCH v5 06/21] csky: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:01 ` [PATCH v5 07/21] ia64: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:01 ` [PATCH v5 08/21] mips: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:01 ` [PATCH v5 09/21] parisc: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:01 ` [PATCH v5 10/21] powerpc: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:02 ` [PATCH v5 11/21] s390: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:02 ` [PATCH v5 12/21] sh: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:02 ` [PATCH v5 13/21] sparc: " Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-08-29 13:02 ` [PATCH v5 14/21] kprobes: Remove NMI context check Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] " tip-bot2 for Masami Hiramatsu
2020-10-31 1:38 ` [PATCH v5 14/21] " Steven Rostedt
2020-11-02 5:11 ` Masami Hiramatsu
2020-11-02 5:53 ` Masami Hiramatsu
2020-11-02 7:02 ` Masami Hiramatsu
2020-11-02 14:27 ` Steven Rostedt
2020-11-03 5:39 ` Masami Hiramatsu
2020-11-03 16:09 ` Steven Rostedt
2020-11-04 2:08 ` Masami Hiramatsu
2020-11-04 14:47 ` Steven Rostedt
2020-11-05 5:15 ` Masami Hiramatsu
2020-08-29 13:02 ` [PATCH v5 15/21] kprobes: Free kretprobe_instance with rcu callback Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] kprobes: Free kretprobe_instance with RCU callback tip-bot2 for Masami Hiramatsu
2020-08-29 13:03 ` [PATCH v5 16/21] kprobes: Make local used functions static Masami Hiramatsu
2020-09-14 17:16 ` [tip: perf/kprobes] kprobes: Make local " tip-bot2 for Masami Hiramatsu
2020-08-29 13:03 ` [PATCH v5 17/21] llist: Add nonatomic __llist_add() and __llist_dell_all() Masami Hiramatsu
2020-10-12 16:24 ` Ingo Molnar
2020-10-14 0:24 ` Masami Hiramatsu
2020-10-12 17:08 ` [tip: perf/kprobes] " tip-bot2 for Peter Zijlstra
2020-08-29 13:03 ` [PATCH v5 18/21] kprobes: Remove kretprobe hash Masami Hiramatsu
2020-10-12 17:08 ` [tip: perf/kprobes] " tip-bot2 for Peter Zijlstra
2020-08-29 13:03 ` [PATCH v5 19/21] asm-generic/atomic: Add try_cmpxchg() fallbacks Masami Hiramatsu
2020-10-12 16:25 ` Ingo Molnar
2020-10-12 17:08 ` [tip: perf/kprobes] " tip-bot2 for Peter Zijlstra
2020-08-29 13:03 ` [PATCH v5 20/21] freelist: Lock less freelist Masami Hiramatsu
2020-10-12 17:08 ` [tip: perf/kprobes] freelist: Implement lockless freelist tip-bot2 for Peter Zijlstra
2020-08-29 13:03 ` [PATCH v5 21/21] kprobes: Replace rp->free_instance with freelist Masami Hiramatsu
2020-10-12 17:08 ` tip-bot2 for Peter Zijlstra [this message]
2020-09-01 19:08 ` [PATCH v5 00/21] kprobes: Unify kretprobe trampoline handlers and make kretprobe lockless Peter Zijlstra
2020-09-02 0:37 ` Masami Hiramatsu
2020-09-02 7:02 ` peterz
2020-09-02 8:17 ` Masami Hiramatsu
2020-09-02 9:36 ` peterz
2020-09-02 13:19 ` Masami Hiramatsu
2020-09-02 13:42 ` peterz
2020-09-03 1:39 ` Masami Hiramatsu
2020-09-03 2:02 ` Masami Hiramatsu
2020-09-07 17:44 ` Frank Ch. Eigler
2020-09-08 2:55 ` Masami Hiramatsu
2020-09-08 10:37 ` peterz
2020-09-08 11:15 ` Eddy_Wu
2020-09-08 11:33 ` peterz
2020-09-08 15:09 ` Masami Hiramatsu
2020-09-09 5:28 ` Masami Hiramatsu
2020-09-11 2:32 ` Masami Hiramatsu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=160252250482.7002.18034659363554741944.tip-bot2@tip-bot2 \
--to=tip-bot2@linutronix.de \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=mhiramat@kernel.org \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).