[patch 33/38] kallsyms: Take callthunks into account

[Thomas Gleixner <tglx@linutronix.de>]

From: Peter Zijlstra <peterz@infradead.org>

Similar to ftrace and bpf call thunks are creating symbols which are
interesting for things like printing stack-traces, perf, live-patching
and things like that.

Add the required functionality to the core and implement it in x86.

Callthunks will report the same function name as their target, but
their module name will be "callthunk" or "callthunk:${modname}" for
modules.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
---
 arch/x86/kernel/callthunks.c |  155 +++++++++++++++++++++++++++++++++++++++++++
 include/linux/kallsyms.h     |   24 ++++++
 kernel/kallsyms.c            |   23 ++++++
 3 files changed, 202 insertions(+)

--- a/arch/x86/kernel/callthunks.c
+++ b/arch/x86/kernel/callthunks.c
@@ -4,6 +4,7 @@
 
 #include <linux/btree.h>
 #include <linux/debugfs.h>
+#include <linux/kallsyms.h>
 #include <linux/memory.h>
 #include <linux/moduleloader.h>
 #include <linux/set_memory.h>
@@ -548,6 +549,160 @@ void *callthunks_translate_call_dest(voi
 	return dest;
 }
 
+static bool is_module_callthunk(void *addr)
+{
+	bool ret = false;
+
+#ifdef CONFIG_MODULES
+	struct module *mod;
+
+	preempt_disable();
+	mod = __module_address((unsigned long)addr);
+	if (mod && within_module_thunk((unsigned long)addr, mod))
+		ret = true;
+	preempt_enable();
+#endif
+	return ret;
+}
+
+static bool is_callthunk(void *addr)
+{
+	if (builtin_layout.base <= addr &&
+	    addr < builtin_layout.base + builtin_layout.size)
+		return true;
+	return is_module_callthunk(addr);
+}
+
+static void *__callthunk_dest(void *addr)
+{
+	unsigned long mask = callthunk_desc.thunk_size - 1;
+	void *thunk;
+
+	thunk = (void *)((unsigned long)addr & ~mask);
+	thunk += callthunk_desc.template_size;
+	return jump_get_dest(thunk);
+}
+
+static void *callthunk_dest(void *addr)
+{
+	if (!is_callthunk(addr))
+		return NULL;
+	return __callthunk_dest(addr);
+}
+
+static void set_modname(char **modname, unsigned long addr)
+{
+	if (!modname || !IS_ENABLED(CONFIG_MODULES))
+		*modname = "callthunk";
+
+#ifdef CONFIG_MODULES
+	else {
+		struct module * mod;
+
+		preempt_disable();
+		mod = __module_address(addr);
+		*modname = mod->callthunk_name;
+		preempt_enable();
+	}
+#endif
+}
+
+const char *
+callthunk_address_lookup(unsigned long addr, unsigned long *size,
+			 unsigned long *off, char **modname, char *sym)
+{
+	unsigned long dest, mask = callthunk_desc.thunk_size - 1;
+	const char *ret;
+
+	if (!thunks_initialized)
+		return NULL;
+
+	dest = (unsigned long)callthunk_dest((void *)addr);
+	if (!dest)
+		return NULL;
+
+	ret = kallsyms_lookup(dest, size, off, modname, sym);
+	if (!ret)
+		return NULL;
+
+	*off = addr & mask;
+	*size = callthunk_desc.thunk_size;
+
+	set_modname(modname, addr);
+	return ret;
+}
+
+static int get_module_thunk(char **modname, struct module_layout **layoutp,
+			    unsigned int symthunk)
+{
+#ifdef CONFIG_MODULES
+	extern struct list_head modules;
+	struct module *mod;
+	unsigned int size;
+
+	symthunk -= (*layoutp)->text_size;
+	list_for_each_entry_rcu(mod, &modules, list) {
+		if (mod->state == MODULE_STATE_UNFORMED)
+			continue;
+
+		*layoutp = &mod->thunk_layout;
+		size = mod->thunk_layout.text_size;
+
+		if (symthunk >= size) {
+			symthunk -= size;
+			continue;
+		}
+		*modname = mod->callthunk_name;
+		return symthunk;
+	}
+#endif
+	return -ERANGE;
+}
+
+int callthunk_get_kallsym(unsigned int symnum, unsigned long *value,
+			  char *type, char *name, char *module_name,
+			  int *exported)
+{
+	int symthunk = symnum * callthunk_desc.thunk_size;
+	struct module_layout *layout = &builtin_layout;
+	char *modname = "callthunk";
+	void *thunk, *dest;
+	int ret = -ERANGE;
+
+	if (!thunks_initialized)
+		return -ERANGE;
+
+	preempt_disable();
+
+	if (symthunk >= layout->text_size) {
+		symthunk = get_module_thunk(&modname, &layout, symthunk);
+		if (symthunk < 0)
+			goto out;
+	}
+
+	thunk = layout->base + symthunk;
+	dest = __callthunk_dest(thunk);
+
+	if (!dest) {
+		strlcpy(name, "(unknown callthunk)", KSYM_NAME_LEN);
+		ret = 0;
+		goto out;
+	}
+
+	ret = lookup_symbol_name((unsigned long)dest, name);
+	if (ret)
+		goto out;
+
+	*value = (unsigned long)thunk;
+	*exported = 0;
+	*type = 't';
+	strlcpy(module_name, modname, MODULE_NAME_LEN);
+
+out:
+	preempt_enable();
+	return ret;
+}
+
 #ifdef CONFIG_MODULES
 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
 					    struct module *mod)
--- a/include/linux/kallsyms.h
+++ b/include/linux/kallsyms.h
@@ -65,6 +65,30 @@ static inline void *dereference_symbol_d
 	return ptr;
 }
 
+#ifdef CONFIG_CALL_THUNKS
+extern const char *
+callthunk_address_lookup(unsigned long addr, unsigned long *size,
+			 unsigned long *off, char **modname, char *sym);
+extern int callthunk_get_kallsym(unsigned int symnum, unsigned long *value,
+				 char *type, char *name, char *module_name,
+				 int *exported);
+#else
+static inline const char *
+callthunk_address_lookup(unsigned long addr, unsigned long *size,
+			 unsigned long *off, char **modname, char *sym)
+{
+	return NULL;
+}
+
+static inline
+int callthunk_get_kallsym(unsigned int symnum, unsigned long *value,
+			  char *type, char *name, char *module_name,
+			  int *exported)
+{
+	return -1;
+}
+#endif
+
 #ifdef CONFIG_KALLSYMS
 int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *,
 				      unsigned long),
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -365,6 +365,10 @@ static const char *kallsyms_lookup_build
 		ret = ftrace_mod_address_lookup(addr, symbolsize,
 						offset, modname, namebuf);
 
+	if (!ret)
+		ret = callthunk_address_lookup(addr, symbolsize,
+					       offset, modname, namebuf);
+
 found:
 	cleanup_symbol_name(namebuf);
 	return ret;
@@ -578,6 +582,7 @@ struct kallsym_iter {
 	loff_t pos_mod_end;
 	loff_t pos_ftrace_mod_end;
 	loff_t pos_bpf_end;
+	loff_t pos_callthunk_end;
 	unsigned long value;
 	unsigned int nameoff; /* If iterating in core kernel symbols. */
 	char type;
@@ -657,6 +662,20 @@ static int get_ksymbol_bpf(struct kallsy
 	return 1;
 }
 
+static int get_ksymbol_callthunk(struct kallsym_iter *iter)
+{
+	int ret = callthunk_get_kallsym(iter->pos - iter->pos_bpf_end,
+					&iter->value, &iter->type,
+					iter->name, iter->module_name,
+					&iter->exported);
+	if (ret < 0) {
+		iter->pos_callthunk_end = iter->pos;
+		return 0;
+	}
+
+	return 1;
+}
+
 /*
  * This uses "__builtin__kprobes" as a module name for symbols for pages
  * allocated for kprobes' purposes, even though "__builtin__kprobes" is not a
@@ -724,6 +743,10 @@ static int update_iter_mod(struct kallsy
 	    get_ksymbol_bpf(iter))
 		return 1;
 
+	if ((!iter->pos_callthunk_end || iter->pos_callthunk_end > pos) &&
+	    get_ksymbol_callthunk(iter))
+		return 1;
+
 	return get_ksymbol_kprobe(iter);
 }