From: <gregkh@linuxfoundation.org>
To: daniel@iogearbox.net, ast@kernel.org, gregkh@linuxfoundation.org
Cc: <stable@vger.kernel.org>, <stable-commits@vger.kernel.org>
Subject: Patch "bpf, x64: implement retpoline for tail call" has been added to the 4.9-stable tree
Date: Fri, 09 Mar 2018 14:21:44 -0800 [thread overview]
Message-ID: <1520634104119162@kroah.com> (raw)
In-Reply-To: <cfd8963c4c57f676177fb2d3a516a4b63cdccde2.1520521792.git.daniel@iogearbox.net>
This is a note to let you know that I've just added the patch titled
bpf, x64: implement retpoline for tail call
to the 4.9-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary
The filename of the patch is:
bpf-x64-implement-retpoline-for-tail-call.patch
and it can be found in the queue-4.9 subdirectory.
If you, or anyone else, feels it should not be added to the stable tree,
please let <stable@vger.kernel.org> know about it.
>From foo@baz Fri Mar 9 14:20:51 PST 2018
From: Daniel Borkmann <daniel@iogearbox.net>
Date: Thu, 8 Mar 2018 16:17:34 +0100
Subject: bpf, x64: implement retpoline for tail call
To: gregkh@linuxfoundation.org
Cc: ast@kernel.org, daniel@iogearbox.net, stable@vger.kernel.org
Message-ID: <cfd8963c4c57f676177fb2d3a516a4b63cdccde2.1520521792.git.daniel@iogearbox.net>
From: Daniel Borkmann <daniel@iogearbox.net>
[ upstream commit a493a87f38cfa48caaa95c9347be2d914c6fdf29 ]
Implement a retpoline [0] for the BPF tail call JIT'ing that converts
the indirect jump via jmp %rax that is used to make the long jump into
another JITed BPF image. Since this is subject to speculative execution,
we need to control the transient instruction sequence here as well
when CONFIG_RETPOLINE is set, and direct it into a pause + lfence loop.
The latter aligns also with what gcc / clang emits (e.g. [1]).
JIT dump after patch:
# bpftool p d x i 1
0: (18) r2 = map[id:1]
2: (b7) r3 = 0
3: (85) call bpf_tail_call#12
4: (b7) r0 = 2
5: (95) exit
With CONFIG_RETPOLINE:
# bpftool p d j i 1
[...]
33: cmp %edx,0x24(%rsi)
36: jbe 0x0000000000000072 |*
38: mov 0x24(%rbp),%eax
3e: cmp $0x20,%eax
41: ja 0x0000000000000072 |
43: add $0x1,%eax
46: mov %eax,0x24(%rbp)
4c: mov 0x90(%rsi,%rdx,8),%rax
54: test %rax,%rax
57: je 0x0000000000000072 |
59: mov 0x28(%rax),%rax
5d: add $0x25,%rax
61: callq 0x000000000000006d |+
66: pause |
68: lfence |
6b: jmp 0x0000000000000066 |
6d: mov %rax,(%rsp) |
71: retq |
72: mov $0x2,%eax
[...]
* relative fall-through jumps in error case
+ retpoline for indirect jump
Without CONFIG_RETPOLINE:
# bpftool p d j i 1
[...]
33: cmp %edx,0x24(%rsi)
36: jbe 0x0000000000000063 |*
38: mov 0x24(%rbp),%eax
3e: cmp $0x20,%eax
41: ja 0x0000000000000063 |
43: add $0x1,%eax
46: mov %eax,0x24(%rbp)
4c: mov 0x90(%rsi,%rdx,8),%rax
54: test %rax,%rax
57: je 0x0000000000000063 |
59: mov 0x28(%rax),%rax
5d: add $0x25,%rax
61: jmpq *%rax |-
63: mov $0x2,%eax
[...]
* relative fall-through jumps in error case
- plain indirect jump as before
[0] https://support.google.com/faqs/answer/7625886
[1] https://github.com/gcc-mirror/gcc/commit/a31e654fa107be968b802786d747e962c2fcdb2b
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
arch/x86/include/asm/nospec-branch.h | 37 +++++++++++++++++++++++++++++++++++
arch/x86/net/bpf_jit_comp.c | 9 ++++----
2 files changed, 42 insertions(+), 4 deletions(-)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -177,4 +177,41 @@ static inline void indirect_branch_predi
}
#endif /* __ASSEMBLY__ */
+
+/*
+ * Below is used in the eBPF JIT compiler and emits the byte sequence
+ * for the following assembly:
+ *
+ * With retpolines configured:
+ *
+ * callq do_rop
+ * spec_trap:
+ * pause
+ * lfence
+ * jmp spec_trap
+ * do_rop:
+ * mov %rax,(%rsp)
+ * retq
+ *
+ * Without retpolines configured:
+ *
+ * jmp *%rax
+ */
+#ifdef CONFIG_RETPOLINE
+# define RETPOLINE_RAX_BPF_JIT_SIZE 17
+# define RETPOLINE_RAX_BPF_JIT() \
+ EMIT1_off32(0xE8, 7); /* callq do_rop */ \
+ /* spec_trap: */ \
+ EMIT2(0xF3, 0x90); /* pause */ \
+ EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
+ EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
+ /* do_rop: */ \
+ EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \
+ EMIT1(0xC3); /* retq */
+#else
+# define RETPOLINE_RAX_BPF_JIT_SIZE 2
+# define RETPOLINE_RAX_BPF_JIT() \
+ EMIT2(0xFF, 0xE0); /* jmp *%rax */
+#endif
+
#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -12,6 +12,7 @@
#include <linux/filter.h>
#include <linux/if_vlan.h>
#include <asm/cacheflush.h>
+#include <asm/nospec-branch.h>
#include <linux/bpf.h>
int bpf_jit_enable __read_mostly;
@@ -281,7 +282,7 @@ static void emit_bpf_tail_call(u8 **ppro
EMIT2(0x89, 0xD2); /* mov edx, edx */
EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */
offsetof(struct bpf_array, map.max_entries));
-#define OFFSET1 43 /* number of bytes to jump */
+#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */
EMIT2(X86_JBE, OFFSET1); /* jbe out */
label1 = cnt;
@@ -290,7 +291,7 @@ static void emit_bpf_tail_call(u8 **ppro
*/
EMIT2_off32(0x8B, 0x85, -STACKSIZE + 36); /* mov eax, dword ptr [rbp - 516] */
EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */
-#define OFFSET2 32
+#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
EMIT2(X86_JA, OFFSET2); /* ja out */
label2 = cnt;
EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
@@ -304,7 +305,7 @@ static void emit_bpf_tail_call(u8 **ppro
* goto out;
*/
EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */
-#define OFFSET3 10
+#define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
EMIT2(X86_JE, OFFSET3); /* je out */
label3 = cnt;
@@ -317,7 +318,7 @@ static void emit_bpf_tail_call(u8 **ppro
* rdi == ctx (1st arg)
* rax == prog->bpf_func + prologue_size
*/
- EMIT2(0xFF, 0xE0); /* jmp rax */
+ RETPOLINE_RAX_BPF_JIT();
/* out: */
BUILD_BUG_ON(cnt - label1 != OFFSET1);
Patches currently in stable-queue which might be from daniel@iogearbox.net are
queue-4.9/bpf-fix-mlock-precharge-on-arraymaps.patch
queue-4.9/bpf-x64-implement-retpoline-for-tail-call.patch
queue-4.9/bpf-arm64-fix-out-of-bounds-access-in-tail-call.patch
queue-4.9/bpf-fix-wrong-exposure-of-map_flags-into-fdinfo-for-lpm.patch
queue-4.9/bpf-ppc64-fix-out-of-bounds-access-in-tail-call.patch
queue-4.9/bpf-add-schedule-points-in-percpu-arrays-management.patch
next prev parent reply other threads:[~2018-03-09 22:21 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-08 15:17 [PATCH stable 4.9 0/6] BPF stable patches Daniel Borkmann
2018-03-08 15:17 ` [PATCH stable 4.9 1/6] bpf: fix wrong exposure of map_flags into fdinfo for lpm Daniel Borkmann
2018-03-09 22:21 ` Patch "bpf: fix wrong exposure of map_flags into fdinfo for lpm" has been added to the 4.9-stable tree gregkh
2018-03-08 15:17 ` [PATCH stable 4.9 2/6] bpf: fix mlock precharge on arraymaps Daniel Borkmann
2018-03-09 22:21 ` Patch "bpf: fix mlock precharge on arraymaps" has been added to the 4.9-stable tree gregkh
2018-03-08 15:17 ` [PATCH stable 4.9 3/6] bpf, x64: implement retpoline for tail call Daniel Borkmann
2018-03-09 22:21 ` gregkh [this message]
2018-03-10 0:05 ` Patch "bpf, x64: implement retpoline for tail call" has been added to the 4.4-stable tree gregkh
2018-03-08 15:17 ` [PATCH stable 4.9 4/6] bpf, arm64: fix out of bounds access in tail call Daniel Borkmann
2018-03-09 22:21 ` Patch "bpf, arm64: fix out of bounds access in tail call" has been added to the 4.9-stable tree gregkh
2018-03-08 15:17 ` [PATCH stable 4.9 5/6] bpf: add schedule points in percpu arrays management Daniel Borkmann
2018-03-09 22:21 ` Patch "bpf: add schedule points in percpu arrays management" has been added to the 4.9-stable tree gregkh
2018-03-08 15:17 ` [PATCH stable 4.9 6/6] bpf, ppc64: fix out of bounds access in tail call Daniel Borkmann
2018-03-09 22:21 ` Patch "bpf, ppc64: fix out of bounds access in tail call" has been added to the 4.9-stable tree gregkh
2018-03-09 22:22 ` [PATCH stable 4.9 0/6] BPF stable patches Greg KH
2018-03-09 22:36 ` Daniel Borkmann
2018-03-10 0:03 ` Greg KH
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1520634104119162@kroah.com \
--to=gregkh@linuxfoundation.org \
--cc=ast@kernel.org \
--cc=daniel@iogearbox.net \
--cc=stable-commits@vger.kernel.org \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.