From: Steven Rostedt <rostedt@goodmis.org>
To: linux-kernel@vger.kernel.org
Cc: Ingo Molnar <mingo@elte.hu>,
Andrew Morton <akpm@linux-foundation.org>,
Jason Baron <jbaron@redhat.com>, "H. Peter Anvin" <hpa@zytor.com>,
Frederic Weisbecker <fweisbec@gmail.com>
Subject: [PATCH 5/5 v2] x86/jump labels: Use etiher 5 byte or 2 byte jumps
Date: Sat, 28 Jan 2012 11:09:30 -0500 [thread overview]
Message-ID: <20120128161627.908330057@goodmis.org> (raw)
In-Reply-To: 20120128160925.275163704@goodmis.org
[-- Attachment #1: Type: text/plain, Size: 6662 bytes --]
From: Steven Rostedt <srostedt@redhat.com>
Have the jump labels add a "jmp" in the assembly instead
of a default nop. This will cause the assembler to put in
either a 2 byte or 5 byte jmp depending on where the target
lable is.
Then at compile time, the update_jump_label code will replace
the jmps with either 2 or 5 byte nops.
On boot up, the code can be examined to see if the jump label
uses either a 2 or 5 byte nop and replace it.
By allowing the jump labels to be 2 bytes, it speeds up the
nops, not only 2 byte nops are faster than 5 byte nops, but also
because it saves on cache foot print.
text data bss dec hex filename
13403667 3666856 2998272 20068795 13239bb ../nobackup/mxtest/vmlinux-old
13398536 3666856 2998272 20063664 13225b0 ../nobackup/mxtest/vmlinux-new
Converting the current v3.2 trace points saved 5,131 bytes.
As more places use jump labels, this will have a bigger savings.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/jump_label.h | 7 +++-
arch/x86/kernel/jump_label.c | 85 ++++++++++++++++++++++++++++--------
3 files changed, 73 insertions(+), 20 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index efb4294..b5004c1 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -61,6 +61,7 @@ config X86
select HAVE_ARCH_KMEMCHECK
select HAVE_USER_RETURN_NOTIFIER
select HAVE_ARCH_JUMP_LABEL
+ select HAVE_BUILD_TIME_JUMP_LABEL
select HAVE_TEXT_POKE_SMP
select HAVE_GENERIC_HARDIRQS
select HAVE_SPARSE_IRQ
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index 7082743..ba12f7e 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -10,6 +10,11 @@
#define JUMP_LABEL_NOP_SIZE 5
+/*
+ * The JUMP_LABEL_INIT_NOP must match the nops used in
+ * scripts/update_jump_label.c. Otherwise the boot time checks
+ * will fail and trigger a BUG() on boot up.
+ */
#ifdef CONFIG_X86_64
# define JUMP_LABEL_INIT_NOP P6_NOP5_ATOMIC
#else
@@ -19,7 +24,7 @@
static __always_inline bool arch_static_branch(struct jump_label_key *key)
{
asm goto("1:"
- ".byte " __stringify(JUMP_LABEL_INIT_NOP) "\n\t"
+ "jmp %l[l_yes]\n"
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index 12a8812..7112a27 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -16,12 +16,20 @@
#ifdef HAVE_JUMP_LABEL
+/* These are the nops added at compile time */
+static const unsigned char nop_short[] = { P6_NOP2 };
+static const unsigned char default_nop[] = { JUMP_LABEL_INIT_NOP };
+
union jump_code_union {
char code[JUMP_LABEL_NOP_SIZE];
struct {
char jump;
int offset;
- } __attribute__((packed));
+ } __packed;
+ struct {
+ char jump_short;
+ char offset_short;
+ } __packed;
};
static void __jump_label_transform(struct jump_entry *entry,
@@ -30,18 +38,33 @@ static void __jump_label_transform(struct jump_entry *entry,
int init)
{
union jump_code_union code;
+ unsigned char nop;
+ unsigned char op;
+ unsigned size;
+ void *ip = (void *)entry->code;
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
- if (type == JUMP_LABEL_ENABLE) {
- /*
- * We are enabling this jump label. If it is not a nop
- * then something must have gone wrong.
- */
- BUG_ON(memcmp((void *)entry->code, ideal_nop, 5) != 0);
+ /* Use probe_kernel_read()? */
+ op = *(unsigned char *)ip;
+ nop = ideal_nops[NOP_ATOMIC5][0];
- code.jump = 0xe9;
- code.offset = entry->target -
- (entry->code + JUMP_LABEL_NOP_SIZE);
+ if (type == JUMP_LABEL_ENABLE) {
+ if (memcmp(ip, nop_short, 2) == 0) {
+ size = 2;
+ code.jump_short = 0xeb;
+ code.offset = entry->target - (entry->code + 2);
+ /* Check for overflow ? */
+ } else if (memcmp(ip, ideal_nop, 5) == 0) {
+ size = JUMP_LABEL_NOP_SIZE;
+ code.jump = 0xe9;
+ code.offset = entry->target - (entry->code + size);
+ } else
+ /*
+ * The location is not a nop that we were expecting,
+ * something went wrong. Crash the box, as something could be
+ * corrupting the kernel.
+ */
+ BUG();
} else {
/*
* We are disabling this jump label. If it is not what
@@ -50,18 +73,44 @@ static void __jump_label_transform(struct jump_entry *entry,
* are converting the default nop to the ideal nop.
*/
if (init) {
- const unsigned char default_nop[] = { JUMP_LABEL_INIT_NOP };
- BUG_ON(memcmp((void *)entry->code, default_nop, 5) != 0);
- } else {
+ /* Ignore short nops, we do not change them */
+ if (memcmp(ip, nop_short, 2) == 0)
+ return;
+
+ /* We are initializing from the default nop */
+ BUG_ON(memcmp(ip, default_nop, 5) != 0);
+
+ /* Set to the ideal nop */
+ size = JUMP_LABEL_NOP_SIZE;
+ memcpy(&code, ideal_nops[NOP_ATOMIC5], size);
+
+ } else if (op == 0xe9) {
+ /* Replace a 5 byte jmp */
+
+ /* Make sure this is what we expected it to be */
code.jump = 0xe9;
code.offset = entry->target -
(entry->code + JUMP_LABEL_NOP_SIZE);
- BUG_ON(memcmp((void *)entry->code, &code, 5) != 0);
- }
- memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+ BUG_ON(memcmp(ip, &code, 5) != 0);
+
+ size = JUMP_LABEL_NOP_SIZE;
+ memcpy(&code, ideal_nops[NOP_ATOMIC5], size);
+ } else if (op == 0xeb) {
+ /* Replace a 2 byte jmp */
+
+ /* Had better be a 2 byte jmp */
+ code.jump_short = 0xeb;
+ code.offset = entry->target - (entry->code + 2);
+ BUG_ON(memcmp(ip, &code, 2) != 0);
+
+ size = 2;
+ memcpy(&code, nop_short, size);
+ } else
+ /* The code was not what we expected! */
+ BUG();
}
- (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+ (*poker)(ip, &code, size);
}
void arch_jump_label_transform(struct jump_entry *entry,
@@ -88,7 +137,6 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
* If it is not, then we need to update the nop to the ideal nop.
*/
if (!once) {
- const unsigned char default_nop[] = { JUMP_LABEL_INIT_NOP };
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
once++;
if (memcmp(ideal_nop, default_nop, 5) != 0)
@@ -97,5 +145,4 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
if (update)
__jump_label_transform(entry, type, text_poke_early, 1);
}
-
#endif
--
1.7.8.3
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 836 bytes --]
next prev parent reply other threads:[~2012-01-28 16:17 UTC|newest]
Thread overview: 22+ messages / expand[flat|nested] mbox.gz Atom feed top
2012-01-28 16:09 [PATCH 0/5 v2] [GIT PULL] x86/jump label: Paranoid checks and 2 or 5 byte nops Steven Rostedt
2012-01-28 16:09 ` [PATCH 1/5 v2] x86/jump-label: Use best default nops for inital jump label calls Steven Rostedt
2012-01-28 16:09 ` [PATCH 2/5 v2] x86/jump-label: Do not bother updating nops if they are correct Steven Rostedt
2012-01-28 16:09 ` [PATCH 3/5 v2] x86/jump-label: Add safety checks to jump label conversions Steven Rostedt
2012-01-28 16:09 ` [PATCH 4/5 v2] jump labels: Add infrastructure to update jump labels at compile time Steven Rostedt
2012-01-28 16:09 ` Steven Rostedt [this message]
2012-01-30 14:48 ` [PATCH 0/5 v2] [GIT PULL] x86/jump label: Paranoid checks and 2 or 5 byte nops Ingo Molnar
2012-01-30 15:07 ` Steven Rostedt
2012-01-31 19:48 ` Ingo Molnar
2012-02-01 5:08 ` Steven Rostedt
2012-02-01 8:05 ` Ingo Molnar
2012-02-01 13:20 ` Steven Rostedt
2012-02-01 15:21 ` Steven Rostedt
2012-02-02 13:51 ` Ingo Molnar
2012-02-02 14:10 ` H. Peter Anvin
2012-02-02 14:31 ` Steven Rostedt
2012-02-02 22:04 ` Steven Rostedt
2012-02-02 22:07 ` H. Peter Anvin
2012-02-02 22:53 ` Steven Rostedt
2012-02-03 7:36 ` Ingo Molnar
2012-02-03 13:31 ` Steven Rostedt
2012-02-05 11:47 ` Ingo Molnar
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20120128161627.908330057@goodmis.org \
--to=rostedt@goodmis.org \
--cc=akpm@linux-foundation.org \
--cc=fweisbec@gmail.com \
--cc=hpa@zytor.com \
--cc=jbaron@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@elte.hu \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).