linux-kernel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: "Chang S. Bae" <chang.seok.bae@intel.com>
To: Andy Lutomirski <luto@kernel.org>,
	"H . Peter Anvin" <hpa@zytor.com>,
	Thomas Gleixner <tglx@linutronix.de>,
	Ingo Molnar <mingo@kernel.org>
Cc: Andi Kleen <ak@linux.intel.com>,
	Dave Hansen <dave.hansen@linux.intel.com>,
	Markus T Metzger <markus.t.metzger@intel.com>,
	"Ravi V . Shankar" <ravi.v.shankar@intel.com>,
	"Chang S . Bae" <chang.seok.bae@intel.com>,
	linux-kernel@vger.kernel.org
Subject: [PATCH V2 09/15] x86/fsgsbase/64: Enable FSGSBASE instructions in helper functions
Date: Thu, 31 May 2018 10:58:39 -0700	[thread overview]
Message-ID: <1527789525-8857-10-git-send-email-chang.seok.bae@intel.com> (raw)
In-Reply-To: <1527789525-8857-1-git-send-email-chang.seok.bae@intel.com>

The helper functions switch on faster access to FS/GS, when
FSGSBASE enabled.

Accessing user GS base needs a couple of SWPAGS. It is avoidable
if the user GS base is copied at kernel entry and updated as
changed, and (actual) GS base is written back at kernel exit.
However, it costs more cycles to do that. The measured
overhead was (almost) offset to the benefit.

Signed-off-by: Chang S. Bae <chang.seok.bae@intel.com>
Reviewed-by: Andi Kleen <ak@linux.intel.com>
Cc: Any Lutomirski <luto@kernel.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/include/asm/fsgsbase.h | 17 ++++------
 arch/x86/kernel/process_64.c    | 75 +++++++++++++++++++++++++++++++++++------
 2 files changed, 72 insertions(+), 20 deletions(-)

diff --git a/arch/x86/include/asm/fsgsbase.h b/arch/x86/include/asm/fsgsbase.h
index ed42015..903c7a0 100644
--- a/arch/x86/include/asm/fsgsbase.h
+++ b/arch/x86/include/asm/fsgsbase.h
@@ -54,26 +54,23 @@ static __always_inline void wrgsbase(unsigned long gsbase)
 			: "memory");
 }
 
+#include <asm/cpufeature.h>
+
 /* Helper functions for reading/writing FS/GS base */
 
 static inline unsigned long read_fsbase(void)
 {
 	unsigned long fsbase;
 
-	rdmsrl(MSR_FS_BASE, fsbase);
+	if (static_cpu_has(X86_FEATURE_FSGSBASE))
+		fsbase = rdfsbase();
+	else
+		rdmsrl(MSR_FS_BASE, fsbase);
 	return fsbase;
 }
 
 void write_fsbase(unsigned long fsbase);
-
-static inline unsigned long read_inactive_gsbase(void)
-{
-	unsigned long gsbase;
-
-	rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
-	return gsbase;
-}
-
+unsigned long read_inactive_gsbase(void);
 void  write_inactive_gsbase(unsigned long gsbase);
 
 #endif /* CONFIG_X86_64 */
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index cebf240..8ba947f 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -154,6 +154,38 @@ enum which_selector {
 };
 
 /*
+ * Interrupts are disabled here.
+ * Out of line to be protected from kprobes.
+ */
+static noinline __kprobes unsigned long rd_inactive_gsbase(void)
+{
+	unsigned long gsbase, flags;
+
+	local_irq_save(flags);
+	native_swapgs();
+	gsbase = rdgsbase();
+	native_swapgs();
+	local_irq_restore(flags);
+
+	return gsbase;
+}
+
+/*
+ * Interrupts are disabled here.
+ * Out of line to be protected from kprobes.
+ */
+static noinline __kprobes void wr_inactive_gsbase(unsigned long gsbase)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	native_swapgs();
+	wrgsbase(gsbase);
+	native_swapgs();
+	local_irq_restore(flags);
+}
+
+/*
  * Saves the FS or GS base for an outgoing thread if FSGSBASE extensions are
  * not available.  The goal is to be reasonably fast on non-FSGSBASE systems.
  * It's forcibly inlined because it'll generate better code and this function
@@ -333,16 +365,35 @@ static unsigned long task_seg_base(struct task_struct *task,
 
 void write_fsbase(unsigned long fsbase)
 {
-	/* set the selector to 0 to not confuse __switch_to */
-	loadseg(FS, 0);
-	wrmsrl(MSR_FS_BASE, fsbase);
+	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+		wrfsbase(fsbase);
+	} else {
+		/* set the selector to 0 to not confuse __switch_to */
+		loadseg(FS, 0);
+		wrmsrl(MSR_FS_BASE, fsbase);
+	}
+}
+
+unsigned long read_inactive_gsbase(void)
+{
+	unsigned long gsbase;
+
+	if (static_cpu_has(X86_FEATURE_FSGSBASE))
+		gsbase = rd_inactive_gsbase();
+	else
+		rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
+	return gsbase;
 }
 
 void write_inactive_gsbase(unsigned long gsbase)
 {
-	/* set the selector to 0 to not confuse __switch_to */
-	loadseg(GS, 0);
-	wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
+	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+		wr_inactive_gsbase(gsbase);
+	} else {
+		/* set the selector to 0 to not confuse __switch_to */
+		loadseg(GS, 0);
+		wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
+	}
 }
 
 unsigned long read_task_fsbase(struct task_struct *task)
@@ -351,7 +402,8 @@ unsigned long read_task_fsbase(struct task_struct *task)
 
 	if (task == current)
 		fsbase = read_fsbase();
-	else if (task->thread.fsindex == 0)
+	else if (static_cpu_has(X86_FEATURE_FSGSBASE) ||
+		 (task->thread.fsindex == 0))
 		fsbase = task->thread.fsbase;
 	else
 		fsbase = task_seg_base(task, task->thread.fsindex);
@@ -365,7 +417,8 @@ unsigned long read_task_gsbase(struct task_struct *task)
 
 	if (task == current)
 		gsbase = read_inactive_gsbase();
-	else if (task->thread.gsindex == 0)
+	else if (static_cpu_has(X86_FEATURE_FSGSBASE) ||
+		 (task->thread.gsindex == 0))
 		gsbase = task->thread.gsbase;
 	else
 		gsbase = task_seg_base(task, task->thread.gsindex);
@@ -388,7 +441,8 @@ int write_task_fsbase(struct task_struct *task, unsigned long fsbase)
 	task->thread.fsbase = fsbase;
 	if (task == current)
 		write_fsbase(fsbase);
-	task->thread.fsindex = 0;
+	if (!static_cpu_has(X86_FEATURE_FSGSBASE))
+		task->thread.fsindex = 0;
 	put_cpu();
 
 	return 0;
@@ -405,7 +459,8 @@ int write_task_gsbase(struct task_struct *task, unsigned long gsbase)
 	task->thread.gsbase = gsbase;
 	if (task == current)
 		write_inactive_gsbase(gsbase);
-	task->thread.gsindex = 0;
+	if (!static_cpu_has(X86_FEATURE_FSGSBASE))
+		task->thread.gsindex = 0;
 	put_cpu();
 
 	return 0;
-- 
2.7.4

  parent reply	other threads:[~2018-05-31 18:01 UTC|newest]

Thread overview: 31+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-05-31 17:58 [PATCH V2 00/15] x86: Enable FSGSBASE instructions Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 01/15] x86/fsgsbase/64: Introduce FS/GS base helper functions Chang S. Bae
2018-05-31 20:14   ` Andy Lutomirski
2018-05-31 21:03     ` Bae, Chang Seok
2018-05-31 17:58 ` [PATCH V2 02/15] x86/fsgsbase/64: Make ptrace read FS/GS base accurately Chang S. Bae
2018-05-31 20:14   ` Andy Lutomirski
2018-05-31 20:31     ` hpa
2018-05-31 20:38       ` Andy Lutomirski
2018-05-31 17:58 ` [PATCH V2 03/15] x86/fsgsbase/64: Use FS/GS base helpers in core dump Chang S. Bae
2018-05-31 20:15   ` Andy Lutomirski
2018-05-31 21:03     ` Bae, Chang Seok
2018-05-31 17:58 ` [PATCH V2 04/15] x86/fsgsbase/64: Factor out load FS/GS segments from __switch_to Chang S. Bae
2018-05-31 20:16   ` Andy Lutomirski
2018-05-31 17:58 ` [PATCH V2 05/15] x86/vdso: Move out the CPU number store Chang S. Bae
2018-05-31 20:25   ` Andy Lutomirski
2018-05-31 21:06     ` Bae, Chang Seok
2018-06-05  7:02   ` [lkp-robot] [x86/vdso] f52001961d: BUG:kernel_hang_in_early-boot_stage,last_printk:Probing_EDD(edd=off_to_disable)...ok kernel test robot
2018-05-31 17:58 ` [PATCH V2 06/15] taint: Add taint for insecure Chang S. Bae
2018-05-31 20:25   ` Andy Lutomirski
2018-05-31 20:50     ` hpa
2018-05-31 17:58 ` [PATCH V2 07/15] x86/fsgsbase/64: Add 'unsafe_fsgsbase' to enable CR4.FSGSBASE Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 08/15] x86/fsgsbase/64: Add intrinsics/macros for FSGSBASE instructions Chang S. Bae
2018-05-31 17:58 ` Chang S. Bae [this message]
2018-05-31 17:58 ` [PATCH V2 10/15] x86/fsgsbase/64: Preserve FS/GS state in __switch_to if FSGSBASE is on Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 11/15] x86/fsgsbase/64: When copying a thread, use FSGSBASE if enabled Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 12/15] x86/fsgsbase/64: Use per-CPU base as GS base on paranoid_entry Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 13/15] x86/fsgsbase/64: Enable FSGSBASE by default and add a chicken bit Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 14/15] x86/elf: Enumerate kernel FSGSBASE capability in AT_HWCAP2 Chang S. Bae
2018-05-31 17:58 ` [PATCH V2 15/15] x86/fsgsbase/64: Add documentation for FSGSBASE Chang S. Bae
2018-05-31 20:37 ` [PATCH V2 00/15] x86: Enable FSGSBASE instructions Andy Lutomirski
2018-05-31 21:11   ` Bae, Chang Seok

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1527789525-8857-10-git-send-email-chang.seok.bae@intel.com \
    --to=chang.seok.bae@intel.com \
    --cc=ak@linux.intel.com \
    --cc=dave.hansen@linux.intel.com \
    --cc=hpa@zytor.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=luto@kernel.org \
    --cc=markus.t.metzger@intel.com \
    --cc=mingo@kernel.org \
    --cc=ravi.v.shankar@intel.com \
    --cc=tglx@linutronix.de \
    --subject='Re: [PATCH V2 09/15] x86/fsgsbase/64: Enable FSGSBASE instructions in helper functions' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).