From: Steven Rostedt <rostedt@goodmis.org>
To: linux-kernel@vger.kernel.org
Cc: Ingo Molnar <mingo@kernel.org>,
Andrew Morton <akpm@linux-foundation.org>,
X86 ML <x86@kernel.org>, Nadav Amit <nadav.amit@gmail.com>,
Andy Lutomirski <luto@kernel.org>,
Dave Hansen <dave.hansen@linux.intel.com>,
Song Liu <songliubraving@fb.com>,
Masami Hiramatsu <mhiramat@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Daniel Bristot de Oliveira <bristot@redhat.com>,
Alexei Starovoitov <alexei.starovoitov@gmail.com>,
Josh Poimboeuf <jpoimboe@redhat.com>
Subject: [PATCH 09/10] ftrace/x86: Add register_ftrace_direct() for custom trampolines
Date: Fri, 08 Nov 2019 16:28:43 -0500 [thread overview]
Message-ID: <20191108213450.891579507@goodmis.org> (raw)
In-Reply-To: 20191108212834.594904349@goodmis.org
From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>
Enable x86 to allow for register_ftrace_direct(), where a custom trampoline
may be called directly from an ftrace mcount/fentry location.
Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/ftrace.h | 13 +++++++++++++
arch/x86/kernel/ftrace.c | 12 ++++++++++++
arch/x86/kernel/ftrace_64.S | 33 ++++++++++++++++++++++++++-------
include/linux/ftrace.h | 6 ++++++
5 files changed, 58 insertions(+), 7 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d6e1faa28c58..329d9c729ba3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -158,6 +158,7 @@ config X86
select HAVE_DMA_CONTIGUOUS
select HAVE_DYNAMIC_FTRACE
select HAVE_DYNAMIC_FTRACE_WITH_REGS
+ select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
select HAVE_EBPF_JIT
select HAVE_EFFICIENT_UNALIGNED_ACCESS
select HAVE_EISA
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index c38a66661576..c2a7458f912c 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -28,6 +28,19 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
return addr;
}
+/*
+ * When a ftrace registered caller is tracing a function that is
+ * also set by a register_ftrace_direct() call, it needs to be
+ * differentiated in the ftrace_caller trampoline. To do this, we
+ * place the direct caller in the ORIG_AX part of pt_regs. This
+ * tells the ftrace_caller that there's a direct caller.
+ */
+static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
+{
+ /* Emulate a call */
+ regs->orig_ax = addr;
+}
+
#ifdef CONFIG_DYNAMIC_FTRACE
struct dyn_arch_ftrace {
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 024c3053dbba..fef283f6341d 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -1042,6 +1042,18 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
if (unlikely(atomic_read(¤t->tracing_graph_pause)))
return;
+ /*
+ * If the return location is actually pointing directly to
+ * the start of a direct trampoline (if we trace the trampoline
+ * it will still be offset by MCOUNT_INSN_SIZE), then the
+ * return address is actually off by one word, and we
+ * need to adjust for that.
+ */
+ if (ftrace_find_direct_func(self_addr + MCOUNT_INSN_SIZE)) {
+ self_addr = *parent;
+ parent++;
+ }
+
/*
* Protect against fault, even if it shouldn't
* happen. This tool is too much intrusive to
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index 809d54397dba..5d946ab40b52 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -88,6 +88,7 @@ EXPORT_SYMBOL(__fentry__)
movq %rdi, RDI(%rsp)
movq %r8, R8(%rsp)
movq %r9, R9(%rsp)
+ movq $0, ORIG_RAX(%rsp)
/*
* Save the original RBP. Even though the mcount ABI does not
* require this, it helps out callers.
@@ -114,7 +115,8 @@ EXPORT_SYMBOL(__fentry__)
subq $MCOUNT_INSN_SIZE, %rdi
.endm
-.macro restore_mcount_regs
+.macro restore_mcount_regs save=0
+
movq R9(%rsp), %r9
movq R8(%rsp), %r8
movq RDI(%rsp), %rdi
@@ -123,10 +125,7 @@ EXPORT_SYMBOL(__fentry__)
movq RCX(%rsp), %rcx
movq RAX(%rsp), %rax
- /* ftrace_regs_caller can modify %rbp */
- movq RBP(%rsp), %rbp
-
- addq $MCOUNT_REG_SIZE, %rsp
+ addq $MCOUNT_REG_SIZE-\save, %rsp
.endm
@@ -228,10 +227,30 @@ GLOBAL(ftrace_regs_call)
movq R10(%rsp), %r10
movq RBX(%rsp), %rbx
- restore_mcount_regs
+ movq RBP(%rsp), %rbp
+
+ movq ORIG_RAX(%rsp), %rax
+ movq %rax, MCOUNT_REG_SIZE-8(%rsp)
+
+ /* If ORIG_RAX is anything but zero, make this a call to that */
+ movq ORIG_RAX(%rsp), %rax
+ cmpq $0, %rax
+ je 1f
+
+ /* Swap the flags with orig_rax */
+ movq MCOUNT_REG_SIZE(%rsp), %rdi
+ movq %rdi, MCOUNT_REG_SIZE-8(%rsp)
+ movq %rax, MCOUNT_REG_SIZE(%rsp)
+
+ restore_mcount_regs 8
+
+ jmp 2f
+
+1: restore_mcount_regs
+
/* Restore flags */
- popfq
+2: popfq
/*
* As this jmp to ftrace_epilogue can be a short jump
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 8b37b8105398..2bc7bd6b8387 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -272,6 +272,12 @@ static inline struct ftrace_direct_func *ftrace_find_direct_func(unsigned long a
* via ftrace (because there's other callbacks besides the
* direct call), can inform the architecture's trampoline that this
* routine has a direct caller, and what the caller is.
+ *
+ * For example, in x86, it returns the direct caller
+ * callback function via the regs->orig_ax parameter.
+ * Then in the ftrace trampoline, if this is set, it makes
+ * the return from the trampoline jump to the direct caller
+ * instead of going back to the function it just traced.
*/
static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs,
unsigned long addr) { }
--
2.23.0
next prev parent reply other threads:[~2019-11-08 21:35 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-11-08 21:28 [PATCH 00/10] ftrace: Add register_ftrace_direct() Steven Rostedt
2019-11-08 21:28 ` [PATCH 01/10] ftrace: Separate out the copying of a ftrace_hash from __ftrace_hash_move() Steven Rostedt
2019-11-08 21:28 ` [PATCH 02/10] ftrace: Separate out functionality from ftrace_location_range() Steven Rostedt
2019-11-08 21:28 ` [PATCH 03/10] ftrace: Add register_ftrace_direct() Steven Rostedt
2019-11-09 2:29 ` Alexei Starovoitov
2019-11-09 12:33 ` Steven Rostedt
2019-11-14 18:29 ` Steven Rostedt
2019-11-14 18:34 ` Alexei Starovoitov
2019-11-14 18:48 ` Steven Rostedt
2019-11-14 19:05 ` Steven Rostedt
2019-11-13 14:13 ` Miroslav Benes
2019-11-13 14:34 ` Steven Rostedt
2019-11-08 21:28 ` [PATCH 04/10] ftrace: Add ftrace_find_direct_func() Steven Rostedt
2019-11-08 21:28 ` [PATCH 05/10] ftrace: Add sample module that uses register_ftrace_direct() Steven Rostedt
2019-11-08 21:28 ` [PATCH 06/10] ftrace/selftest: Add tests to test register_ftrace_direct() Steven Rostedt
2019-11-08 21:28 ` [PATCH 07/10] ftrace: Add another example of register_ftrace_direct() use case Steven Rostedt
2019-11-08 21:28 ` [PATCH 08/10] ftrace/selftests: Update the direct call selftests to test two direct calls Steven Rostedt
2019-11-08 21:28 ` Steven Rostedt [this message]
2019-11-14 15:34 ` [PATCH 09/10] ftrace/x86: Add register_ftrace_direct() for custom trampolines Miroslav Benes
2019-11-14 16:19 ` Steven Rostedt
2019-11-15 9:32 ` Miroslav Benes
2019-11-08 21:28 ` [PATCH 10/10] ftrace/x86: Add a counter to test function_graph with direct Steven Rostedt
2019-11-08 22:51 ` [PATCH 00/10] ftrace: Add register_ftrace_direct() Josh Poimboeuf
2019-11-09 1:00 ` Steven Rostedt
2019-11-11 8:47 ` Peter Zijlstra
2019-11-11 14:15 ` Josh Poimboeuf
2019-11-13 15:10 ` Miroslav Benes
2019-11-13 16:31 ` Steven Rostedt
2019-11-14 9:05 ` Miroslav Benes
2019-11-14 14:36 ` Steven Rostedt
2019-11-14 15:42 ` Miroslav Benes
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191108213450.891579507@goodmis.org \
--to=rostedt@goodmis.org \
--cc=akpm@linux-foundation.org \
--cc=alexei.starovoitov@gmail.com \
--cc=bristot@redhat.com \
--cc=dave.hansen@linux.intel.com \
--cc=jpoimboe@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mhiramat@kernel.org \
--cc=mingo@kernel.org \
--cc=nadav.amit@gmail.com \
--cc=peterz@infradead.org \
--cc=songliubraving@fb.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).