[PATCH 03/13] kprobe: Add support to register multiple ftrace kprobes

[Jiri Olsa <jolsa@redhat.com>]

Adding support to register kprobe on multiple addresses within
single kprobe object instance.

It uses the CONFIG_KPROBES_ON_FTRACE feature (so it's only
available for function entry addresses) and separated ftrace_ops
object placed directly in the kprobe object.

There's new CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE config option,
enabled for archs that support multi kprobes.

It registers all the provided addresses in the ftrace_ops object
filter and adds separate ftrace_ops callback.

To register multi kprobe user provides array of addresses or
symbols with their count, like:

  struct kprobe kp = {};

  kp.multi.symbols = (const char **) symbols;
  kp.multi.cnt = cnt;
  ...

  err = register_kprobe(&kp);

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 arch/Kconfig                     |   3 +
 arch/x86/Kconfig                 |   1 +
 arch/x86/kernel/kprobes/ftrace.c |  48 ++++++--
 include/linux/kprobes.h          |  25 ++++
 kernel/kprobes.c                 | 204 +++++++++++++++++++++++++------
 5 files changed, 235 insertions(+), 46 deletions(-)

diff --git a/arch/Kconfig b/arch/Kconfig
index d3c4ab249e9c..0131636e1ef8 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -191,6 +191,9 @@ config HAVE_OPTPROBES
 config HAVE_KPROBES_ON_FTRACE
 	bool
 
+config HAVE_KPROBES_MULTI_ON_FTRACE
+	bool
+
 config ARCH_CORRECT_STACKTRACE_ON_KRETPROBE
 	bool
 	help
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 5c2ccb85f2ef..0c870238016a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -217,6 +217,7 @@ config X86
 	select HAVE_KERNEL_ZSTD
 	select HAVE_KPROBES
 	select HAVE_KPROBES_ON_FTRACE
+	select HAVE_KPROBES_MULTI_ON_FTRACE
 	select HAVE_FUNCTION_ERROR_INJECTION
 	select HAVE_KRETPROBES
 	select HAVE_KVM
diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c
index dd2ec14adb77..ac4d256b89c6 100644
--- a/arch/x86/kernel/kprobes/ftrace.c
+++ b/arch/x86/kernel/kprobes/ftrace.c
@@ -12,22 +12,14 @@
 
 #include "common.h"
 
-/* Ftrace callback handler for kprobes -- called under preempt disabled */
-void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
-			   struct ftrace_ops *ops, struct ftrace_regs *fregs)
+static void ftrace_handler(struct kprobe *p, unsigned long ip,
+			   struct ftrace_regs *fregs)
 {
 	struct pt_regs *regs = ftrace_get_regs(fregs);
-	struct kprobe *p;
 	struct kprobe_ctlblk *kcb;
-	int bit;
 
-	bit = ftrace_test_recursion_trylock(ip, parent_ip);
-	if (bit < 0)
-		return;
-
-	p = get_kprobe((kprobe_opcode_t *)ip);
 	if (unlikely(!p) || kprobe_disabled(p))
-		goto out;
+		return;
 
 	kcb = get_kprobe_ctlblk();
 	if (kprobe_running()) {
@@ -57,11 +49,43 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
 		 */
 		__this_cpu_write(current_kprobe, NULL);
 	}
-out:
+}
+
+/* Ftrace callback handler for kprobes -- called under preempt disabled */
+void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
+			   struct ftrace_ops *ops, struct ftrace_regs *fregs)
+{
+	struct kprobe *p;
+	int bit;
+
+	bit = ftrace_test_recursion_trylock(ip, parent_ip);
+	if (bit < 0)
+		return;
+
+	p = get_kprobe((kprobe_opcode_t *)ip);
+	ftrace_handler(p, ip, fregs);
+
 	ftrace_test_recursion_unlock(bit);
 }
 NOKPROBE_SYMBOL(kprobe_ftrace_handler);
 
+void kprobe_ftrace_multi_handler(unsigned long ip, unsigned long parent_ip,
+				 struct ftrace_ops *ops, struct ftrace_regs *fregs)
+{
+	struct kprobe *p;
+	int bit;
+
+	bit = ftrace_test_recursion_trylock(ip, parent_ip);
+	if (bit < 0)
+		return;
+
+	p = container_of(ops, struct kprobe, multi.ops);
+	ftrace_handler(p, ip, fregs);
+
+	ftrace_test_recursion_unlock(bit);
+}
+NOKPROBE_SYMBOL(kprobe_ftrace_multi_handler);
+
 int arch_prepare_kprobe_ftrace(struct kprobe *p)
 {
 	p->ainsn.insn = NULL;
diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
index a204df4fef96..03fd86ef69cb 100644
--- a/include/linux/kprobes.h
+++ b/include/linux/kprobes.h
@@ -68,6 +68,16 @@ struct kprobe {
 	/* location of the probe point */
 	kprobe_opcode_t *addr;
 
+#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE
+	/* location of the multi probe points */
+	struct {
+		const char **symbols;
+		kprobe_opcode_t **addrs;
+		unsigned int cnt;
+		struct ftrace_ops ops;
+	} multi;
+#endif
+
 	/* Allow user to indicate symbol name of the probe point */
 	const char *symbol_name;
 
@@ -105,6 +115,7 @@ struct kprobe {
 				   * this flag is only for optimized_kprobe.
 				   */
 #define KPROBE_FLAG_FTRACE	8 /* probe is using ftrace */
+#define KPROBE_FLAG_MULTI      16 /* probe multi addresses */
 
 /* Has this kprobe gone ? */
 static inline bool kprobe_gone(struct kprobe *p)
@@ -130,6 +141,18 @@ static inline bool kprobe_ftrace(struct kprobe *p)
 	return p->flags & KPROBE_FLAG_FTRACE;
 }
 
+/* Is this ftrace multi kprobe ? */
+static inline bool kprobe_ftrace_multi(struct kprobe *p)
+{
+	return kprobe_ftrace(p) && (p->flags & KPROBE_FLAG_MULTI);
+}
+
+/* Is this single kprobe ? */
+static inline bool kprobe_single(struct kprobe *p)
+{
+	return !(p->flags & KPROBE_FLAG_MULTI);
+}
+
 /*
  * Function-return probe -
  * Note:
@@ -365,6 +388,8 @@ static inline void wait_for_kprobe_optimizer(void) { }
 #ifdef CONFIG_KPROBES_ON_FTRACE
 extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
 				  struct ftrace_ops *ops, struct ftrace_regs *fregs);
+extern void kprobe_ftrace_multi_handler(unsigned long ip, unsigned long parent_ip,
+					struct ftrace_ops *ops, struct ftrace_regs *fregs);
 extern int arch_prepare_kprobe_ftrace(struct kprobe *p);
 #else
 static inline int arch_prepare_kprobe_ftrace(struct kprobe *p)
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index c4060a8da050..e7729e20d85c 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -44,6 +44,7 @@
 #include <asm/cacheflush.h>
 #include <asm/errno.h>
 #include <linux/uaccess.h>
+#include <linux/ftrace.h>
 
 #define KPROBE_HASH_BITS 6
 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
@@ -1022,6 +1023,35 @@ static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 }
 #endif /* CONFIG_OPTPROBES */
 
+static int check_kprobe_address(unsigned long addr)
+{
+	/* Ensure it is not in reserved area nor out of text */
+	return !kernel_text_address(addr) ||
+		within_kprobe_blacklist(addr) ||
+		jump_label_text_reserved((void *) addr, (void *) addr) ||
+		static_call_text_reserved((void *) addr, (void *) addr) ||
+		find_bug(addr);
+}
+
+static int check_ftrace_location(unsigned long addr, struct kprobe *p)
+{
+	unsigned long ftrace_addr;
+
+	ftrace_addr = ftrace_location(addr);
+	if (ftrace_addr) {
+#ifdef CONFIG_KPROBES_ON_FTRACE
+		/* Given address is not on the instruction boundary */
+		if (addr != ftrace_addr)
+			return -EILSEQ;
+		if (p)
+			p->flags |= KPROBE_FLAG_FTRACE;
+#else	/* !CONFIG_KPROBES_ON_FTRACE */
+		return -EINVAL;
+#endif
+	}
+	return 0;
+}
+
 #ifdef CONFIG_KPROBES_ON_FTRACE
 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
 	.func = kprobe_ftrace_handler,
@@ -1043,6 +1073,13 @@ static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
 
 	lockdep_assert_held(&kprobe_mutex);
 
+#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE
+	if (kprobe_ftrace_multi(p)) {
+		ret = register_ftrace_function(&p->multi.ops);
+		WARN(ret < 0, "Failed to register kprobe-multi-ftrace (error %d)\n", ret);
+		return ret;
+	}
+#endif
 	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
 	if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret))
 		return ret;
@@ -1081,6 +1118,13 @@ static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
 
 	lockdep_assert_held(&kprobe_mutex);
 
+#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE
+	if (kprobe_ftrace_multi(p)) {
+		ret = unregister_ftrace_function(&p->multi.ops);
+		WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret);
+		return ret;
+	}
+#endif
 	if (*cnt == 1) {
 		ret = unregister_ftrace_function(ops);
 		if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
@@ -1103,6 +1147,94 @@ static int disarm_kprobe_ftrace(struct kprobe *p)
 		ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
 		ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
 }
+
+#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE
+/*
+ * In addition to standard kprobe address check for multi
+ * ftrace kprobes we also allow only:
+ * - ftrace managed function entry address
+ * - kernel core only address
+ */
+static unsigned long check_ftrace_addr(unsigned long addr)
+{
+	int err;
+
+	if (!addr)
+		return -EINVAL;
+	err = check_ftrace_location(addr, NULL);
+	if (err)
+		return err;
+	if (check_kprobe_address(addr))
+		return -EINVAL;
+	if (__module_text_address(addr))
+		return -EINVAL;
+	return 0;
+}
+
+static int check_ftrace_multi(struct kprobe *p)
+{
+	kprobe_opcode_t **addrs = p->multi.addrs;
+	const char **symbols = p->multi.symbols;
+	unsigned int i, cnt = p->multi.cnt;
+	unsigned long addr, *ips;
+	int err;
+
+	if ((symbols && addrs) || (!symbols && !addrs))
+		return -EINVAL;
+
+	/* do we want sysctl for this? */
+	if (cnt >= 20000)
+		return -E2BIG;
+
+	ips = kmalloc(sizeof(*ips) * cnt, GFP_KERNEL);
+	if (!ips)
+		return -ENOMEM;
+
+	for (i = 0; i < cnt; i++) {
+		if (symbols)
+			addr = (unsigned long) kprobe_lookup_name(symbols[i], 0);
+		else
+			addr = (unsigned long) addrs[i];
+		ips[i] = addr;
+	}
+
+	jump_label_lock();
+	preempt_disable();
+
+	for (i = 0; i < cnt; i++) {
+		err = check_ftrace_addr(ips[i]);
+		if (err)
+			break;
+	}
+
+	preempt_enable();
+	jump_label_unlock();
+
+	if (err)
+		goto out;
+
+	err = ftrace_set_filter_ips(&p->multi.ops, ips, cnt, 0, 0);
+	if (err)
+		goto out;
+
+	p->multi.ops.func = kprobe_ftrace_multi_handler;
+	p->multi.ops.flags = FTRACE_OPS_FL_SAVE_REGS|FTRACE_OPS_FL_DYNAMIC;
+
+	p->flags |= KPROBE_FLAG_MULTI|KPROBE_FLAG_FTRACE;
+	if (p->post_handler)
+		p->multi.ops.flags |= FTRACE_OPS_FL_IPMODIFY;
+
+out:
+	kfree(ips);
+	return err;
+}
+
+static void free_ftrace_multi(struct kprobe *p)
+{
+	ftrace_free_filter(&p->multi.ops);
+}
+#endif
+
 #else	/* !CONFIG_KPROBES_ON_FTRACE */
 static inline int arm_kprobe_ftrace(struct kprobe *p)
 {
@@ -1489,6 +1621,9 @@ static struct kprobe *__get_valid_kprobe(struct kprobe *p)
 
 	lockdep_assert_held(&kprobe_mutex);
 
+	if (kprobe_ftrace_multi(p))
+		return p;
+
 	ap = get_kprobe(p->addr);
 	if (unlikely(!ap))
 		return NULL;
@@ -1520,41 +1655,18 @@ static inline int warn_kprobe_rereg(struct kprobe *p)
 	return ret;
 }
 
-static int check_ftrace_location(struct kprobe *p)
-{
-	unsigned long ftrace_addr;
-
-	ftrace_addr = ftrace_location((unsigned long)p->addr);
-	if (ftrace_addr) {
-#ifdef CONFIG_KPROBES_ON_FTRACE
-		/* Given address is not on the instruction boundary */
-		if ((unsigned long)p->addr != ftrace_addr)
-			return -EILSEQ;
-		p->flags |= KPROBE_FLAG_FTRACE;
-#else	/* !CONFIG_KPROBES_ON_FTRACE */
-		return -EINVAL;
-#endif
-	}
-	return 0;
-}
-
 static int check_kprobe_address_safe(struct kprobe *p,
 				     struct module **probed_mod)
 {
 	int ret;
 
-	ret = check_ftrace_location(p);
+	ret = check_ftrace_location((unsigned long) p->addr, p);
 	if (ret)
 		return ret;
 	jump_label_lock();
 	preempt_disable();
 
-	/* Ensure it is not in reserved area nor out of text */
-	if (!kernel_text_address((unsigned long) p->addr) ||
-	    within_kprobe_blacklist((unsigned long) p->addr) ||
-	    jump_label_text_reserved(p->addr, p->addr) ||
-	    static_call_text_reserved(p->addr, p->addr) ||
-	    find_bug((unsigned long)p->addr)) {
+	if (check_kprobe_address((unsigned long) p->addr)) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -1599,13 +1711,16 @@ static unsigned long resolve_func_addr(kprobe_opcode_t *addr)
 	return 0;
 }
 
-int register_kprobe(struct kprobe *p)
+static int check_addr(struct kprobe *p, struct module **probed_mod)
 {
 	int ret;
-	struct kprobe *old_p;
-	struct module *probed_mod;
 	kprobe_opcode_t *addr;
 
+#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE
+	if (p->multi.cnt)
+		return check_ftrace_multi(p);
+#endif
+
 	/* Adjust probe address from symbol */
 	addr = kprobe_addr(p);
 	if (IS_ERR(addr))
@@ -1616,13 +1731,21 @@ int register_kprobe(struct kprobe *p)
 	ret = warn_kprobe_rereg(p);
 	if (ret)
 		return ret;
+	return check_kprobe_address_safe(p, probed_mod);
+}
+
+int register_kprobe(struct kprobe *p)
+{
+	struct module *probed_mod = NULL;
+	struct kprobe *old_p;
+	int ret;
 
 	/* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
 	p->flags &= KPROBE_FLAG_DISABLED;
 	p->nmissed = 0;
 	INIT_LIST_HEAD(&p->list);
 
-	ret = check_kprobe_address_safe(p, &probed_mod);
+	ret = check_addr(p, &probed_mod);
 	if (ret)
 		return ret;
 
@@ -1644,14 +1767,21 @@ int register_kprobe(struct kprobe *p)
 	if (ret)
 		goto out;
 
-	INIT_HLIST_NODE(&p->hlist);
-	hlist_add_head_rcu(&p->hlist,
-		       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
+	/*
+	 * Multi ftrace kprobes do not have single address,
+	 * so they are not stored in the kprobe_table hash.
+	 */
+	if (kprobe_single(p)) {
+		INIT_HLIST_NODE(&p->hlist);
+		hlist_add_head_rcu(&p->hlist,
+			       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
+	}
 
 	if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
 		ret = arm_kprobe(p);
 		if (ret) {
-			hlist_del_rcu(&p->hlist);
+			if (kprobe_single(p))
+				hlist_del_rcu(&p->hlist);
 			synchronize_rcu();
 			goto out;
 		}
@@ -1778,7 +1908,13 @@ static int __unregister_kprobe_top(struct kprobe *p)
 	return 0;
 
 disarmed:
-	hlist_del_rcu(&ap->hlist);
+	if (kprobe_single(ap))
+		hlist_del_rcu(&ap->hlist);
+
+#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE
+	if (kprobe_ftrace_multi(ap))
+		free_ftrace_multi(ap);
+#endif
 	return 0;
 }
 
-- 
2.33.1