From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: x86@kernel.org, Linus Torvalds <torvalds@linuxfoundation.org>,
Andy Lutomirski <luto@kernel.org>,
Stephen Hemminger <stephen@networkplumber.org>,
Willy Tarreau <w@1wt.eu>, Juergen Gross <jgross@suse.com>,
Sean Christopherson <sean.j.christopherson@intel.com>,
"H. Peter Anvin" <hpa@zytor.com>
Subject: [patch V2 06/16] x86/io: Speedup schedule out of I/O bitmap user
Date: Mon, 11 Nov 2019 23:03:20 +0100 [thread overview]
Message-ID: <20191111223052.086299881@linutronix.de> (raw)
In-Reply-To: 20191111220314.519933535@linutronix.de
From: Thomas Gleixner <tglx@linutronix.de>
There is no requirement to update the TSS I/O bitmap when a thread using it is
scheduled out and the incoming thread does not use it.
For the permission check based on the TSS I/O bitmap the CPU calculates the memory
location of the I/O bitmap by the address of the TSS and the io_bitmap_base member
of the tss_struct. The easiest way to invalidate the I/O bitmap is to switch the
offset to an address outside of the TSS limit.
If an I/O instruction is issued from user space the TSS limit causes #GP to be
raised in the same was as valid I/O bitmap with all bits set to 1 would do.
This removes the extra work when an I/O bitmap using task is scheduled out
and puts the burden on the rare I/O bitmap users when they are scheduled
in.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
arch/x86/include/asm/processor.h | 38 ++++++++++++++++-------
arch/x86/kernel/cpu/common.c | 3 +
arch/x86/kernel/doublefault.c | 2 -
arch/x86/kernel/ioport.c | 7 +++-
arch/x86/kernel/process.c | 63 ++++++++++++++++++++++-----------------
5 files changed, 70 insertions(+), 43 deletions(-)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -330,8 +330,23 @@ struct x86_hw_tss {
#define IO_BITMAP_BITS 65536
#define IO_BITMAP_BYTES (IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long))
-#define IO_BITMAP_OFFSET (offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))
-#define INVALID_IO_BITMAP_OFFSET 0x8000
+
+#define IO_BITMAP_OFFSET_VALID \
+ (offsetof(struct tss_struct, io_bitmap) - \
+ offsetof(struct tss_struct, x86_tss))
+
+/*
+ * sizeof(unsigned long) coming from an extra "long" at the end
+ * of the iobitmap.
+ *
+ * -1? seg base+limit should be pointing to the address of the
+ * last valid byte
+ */
+#define __KERNEL_TSS_LIMIT \
+ (IO_BITMAP_OFFSET_VALID + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)
+
+/* Base offset outside of TSS_LIMIT so unpriviledged IO causes #GP */
+#define IO_BITMAP_OFFSET_INVALID (__KERNEL_TSS_LIMIT + 1)
struct entry_stack {
unsigned long words[64];
@@ -350,6 +365,15 @@ struct tss_struct {
struct x86_hw_tss x86_tss;
/*
+ * Store the dirty size of the last io bitmap offender. The next
+ * one will have to do the cleanup as the switch out to a non io
+ * bitmap user will just set x86_tss.io_bitmap_base to a value
+ * outside of the TSS limit. So for sane tasks there is no need to
+ * actually touch the io_bitmap at all.
+ */
+ unsigned int io_bitmap_prev_max;
+
+ /*
* The extra 1 is there because the CPU will access an
* additional byte beyond the end of the IO permission
* bitmap. The extra byte must be all 1 bits, and must
@@ -360,16 +384,6 @@ struct tss_struct {
DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);
-/*
- * sizeof(unsigned long) coming from an extra "long" at the end
- * of the iobitmap.
- *
- * -1? seg base+limit should be pointing to the address of the
- * last valid byte
- */
-#define __KERNEL_TSS_LIMIT \
- (IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)
-
/* Per CPU interrupt stacks */
struct irq_stack {
char stack[IRQ_STACK_SIZE];
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1860,7 +1860,8 @@ void cpu_init(void)
/* Initialize the TSS. */
tss_setup_ist(tss);
- tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
+ tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
+ tss->io_bitmap_prev_max = 0;
memset(tss->io_bitmap, 0xff, sizeof(tss->io_bitmap));
set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
--- a/arch/x86/kernel/doublefault.c
+++ b/arch/x86/kernel/doublefault.c
@@ -54,7 +54,7 @@ struct x86_hw_tss doublefault_tss __cach
.sp0 = STACK_START,
.ss0 = __KERNEL_DS,
.ldt = 0,
- .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
+ .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
.ip = (unsigned long) doublefault_fn,
/* 0x2 bit is always set */
--- a/arch/x86/kernel/ioport.c
+++ b/arch/x86/kernel/ioport.c
@@ -40,8 +40,6 @@ long ksys_ioperm(unsigned long from, uns
return -ENOMEM;
memset(bitmap, 0xff, IO_BITMAP_BYTES);
- t->io_bitmap_ptr = bitmap;
- set_thread_flag(TIF_IO_BITMAP);
/*
* Now that we have an IO bitmap, we need our TSS limit to be
@@ -50,6 +48,11 @@ long ksys_ioperm(unsigned long from, uns
* limit correct.
*/
preempt_disable();
+ t->io_bitmap_ptr = bitmap;
+ set_thread_flag(TIF_IO_BITMAP);
+ /* Make the bitmap base in the TSS valid */
+ tss = this_cpu_ptr(&cpu_tss_rw);
+ tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_VALID;
refresh_tss_limit();
preempt_enable();
}
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -72,18 +72,9 @@
#ifdef CONFIG_X86_32
.ss0 = __KERNEL_DS,
.ss1 = __KERNEL_CS,
- .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
#endif
+ .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
},
-#ifdef CONFIG_X86_32
- /*
- * Note that the .io_bitmap member must be extra-big. This is because
- * the CPU will access an additional byte beyond the end of the IO
- * permission bitmap. The extra byte must be all 1 bits, and must
- * be within the limit.
- */
- .io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 },
-#endif
};
EXPORT_PER_CPU_SYMBOL(cpu_tss_rw);
@@ -112,18 +103,18 @@ void exit_thread(struct task_struct *tsk
struct thread_struct *t = &tsk->thread;
unsigned long *bp = t->io_bitmap_ptr;
struct fpu *fpu = &t->fpu;
+ struct tss_struct *tss;
if (bp) {
- struct tss_struct *tss = &per_cpu(cpu_tss_rw, get_cpu());
+ preempt_disable();
+ tss = this_cpu_ptr(&cpu_tss_rw);
t->io_bitmap_ptr = NULL;
- clear_thread_flag(TIF_IO_BITMAP);
- /*
- * Careful, clear this in the TSS too:
- */
- memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
t->io_bitmap_max = 0;
- put_cpu();
+ clear_thread_flag(TIF_IO_BITMAP);
+ /* Invalidate the io bitmap base in the TSS */
+ tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
+ preempt_enable();
kfree(bp);
}
@@ -363,29 +354,47 @@ void arch_setup_new_exec(void)
}
}
-static inline void switch_to_bitmap(struct thread_struct *prev,
- struct thread_struct *next,
+static inline void switch_to_bitmap(struct thread_struct *next,
unsigned long tifp, unsigned long tifn)
{
struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
if (tifn & _TIF_IO_BITMAP) {
/*
- * Copy the relevant range of the IO bitmap.
- * Normally this is 128 bytes or less:
+ * Copy at least the size of the incoming tasks bitmap
+ * which covers the last permitted I/O port.
+ *
+ * If the previous task which used an io bitmap had more
+ * bits permitted, then the copy needs to cover those as
+ * well so they get turned off.
*/
memcpy(tss->io_bitmap, next->io_bitmap_ptr,
- max(prev->io_bitmap_max, next->io_bitmap_max));
+ max(tss->io_bitmap_prev_max, next->io_bitmap_max));
+
+ /* Store the new max and set io_bitmap_base valid */
+ tss->io_bitmap_prev_max = next->io_bitmap_max;
+ tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_VALID;
+
/*
- * Make sure that the TSS limit is correct for the CPU
- * to notice the IO bitmap.
+ * Make sure that the TSS limit is covering the io bitmap.
+ * It might have been cut down by a VMEXIT to 0x67 which
+ * would cause a subsequent I/O access from user space to
+ * trigger a #GP because tbe bitmap is outside the TSS
+ * limit.
*/
refresh_tss_limit();
} else if (tifp & _TIF_IO_BITMAP) {
/*
- * Clear any possible leftover bits:
+ * Do not touch the bitmap. Let the next bitmap using task
+ * deal with the mess. Just make the io_bitmap_base invalid
+ * by moving it outside the TSS limit so any subsequent I/O
+ * access from user space will trigger a #GP.
+ *
+ * This is correct even when VMEXIT rewrites the TSS limit
+ * to 0x67 as the only requirement is that the base points
+ * outside the limit.
*/
- memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
+ tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
}
}
@@ -599,7 +608,7 @@ void __switch_to_xtra(struct task_struct
tifn = READ_ONCE(task_thread_info(next_p)->flags);
tifp = READ_ONCE(task_thread_info(prev_p)->flags);
- switch_to_bitmap(prev, next, tifp, tifn);
+ switch_to_bitmap(next, tifp, tifn);
propagate_user_return_notify(prev_p, next_p);
next prev parent reply other threads:[~2019-11-11 22:35 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-11-11 22:03 [patch V2 00/16] x86/iopl: Prevent user space from using CLI/STI with iopl(3) Thomas Gleixner
2019-11-11 22:03 ` [patch V2 01/16] x86/ptrace: Prevent truncation of bitmap size Thomas Gleixner
2019-11-12 15:34 ` Andy Lutomirski
2019-11-11 22:03 ` [patch V2 02/16] x86/process: Unify copy_thread_tls() Thomas Gleixner
2019-11-11 22:03 ` [patch V2 03/16] x86/cpu: Unify cpu_init() Thomas Gleixner
2019-11-11 22:03 ` [patch V2 04/16] x86/tss: Fix and move VMX BUILD_BUG_ON() Thomas Gleixner
2019-11-11 22:44 ` Paolo Bonzini
2019-11-12 15:37 ` Andy Lutomirski
2019-11-11 22:03 ` [patch V2 05/16] x86/iopl: Cleanup include maze Thomas Gleixner
2019-11-12 15:37 ` Andy Lutomirski
2019-11-11 22:03 ` Thomas Gleixner [this message]
2019-11-12 16:00 ` [patch V2 06/16] x86/io: Speedup schedule out of I/O bitmap user Andy Lutomirski
2019-11-12 17:08 ` Thomas Gleixner
2019-11-11 22:03 ` [patch V2 07/16] x86/ioperm: Move iobitmap data into a struct Thomas Gleixner
2019-11-12 16:02 ` Andy Lutomirski
2019-11-12 17:08 ` Thomas Gleixner
2019-11-11 22:03 ` [patch V2 08/16] x86/ioperm: Add bitmap sequence number Thomas Gleixner
2019-11-12 9:22 ` Peter Zijlstra
2019-11-12 9:55 ` [patch V2 08/16] x86/ioperm: Add bitmap sequence numberc Thomas Gleixner
2019-11-12 16:08 ` [patch V2 08/16] x86/ioperm: Add bitmap sequence number Andy Lutomirski
2019-11-12 17:10 ` Thomas Gleixner
2019-11-11 22:03 ` [patch V2 09/16] x86/ioperm: Move TSS bitmap update to exit to user work Thomas Gleixner
2019-11-12 16:16 ` Andy Lutomirski
2019-11-12 17:20 ` Thomas Gleixner
2019-11-12 17:41 ` Andy Lutomirski
2019-11-12 17:46 ` Linus Torvalds
2019-11-13 8:30 ` Peter Zijlstra
2019-11-11 22:03 ` [patch V2 10/16] x86/ioperm: Remove bitmap if all permissions dropped Thomas Gleixner
2019-11-12 17:43 ` Andy Lutomirski
2019-11-11 22:03 ` [patch V2 11/16] x86/ioperm: Share I/O bitmap if identical Thomas Gleixner
2019-11-12 7:14 ` Ingo Molnar
2019-11-12 7:17 ` Thomas Gleixner
2019-11-12 7:52 ` Ingo Molnar
2019-11-12 9:15 ` Peter Zijlstra
2019-11-12 9:51 ` Thomas Gleixner
2019-11-14 11:02 ` David Laight
2019-11-14 12:39 ` Thomas Gleixner
2019-11-14 13:09 ` Peter Zijlstra
2019-11-14 13:22 ` David Laight
2019-11-12 18:12 ` Andy Lutomirski
2019-11-11 22:03 ` [patch V2 12/16] selftests/x86/ioperm: Extend testing so the shared bitmap is exercised Thomas Gleixner
2019-11-11 22:03 ` [patch V2 13/16] x86/iopl: Fixup misleading comment Thomas Gleixner
2019-11-12 18:14 ` Andy Lutomirski
2019-11-11 22:03 ` [patch V2 14/16] x86/iopl: Restrict iopl() permission scope Thomas Gleixner
2019-11-11 23:03 ` Thomas Gleixner
2019-11-12 6:32 ` Ingo Molnar
2019-11-12 8:42 ` Ingo Molnar
2019-11-12 10:07 ` Thomas Gleixner
2019-11-12 18:35 ` Andy Lutomirski
2019-11-11 22:03 ` [patch V2 15/16] x86/iopl: Remove legacy IOPL option Thomas Gleixner
2019-11-12 18:37 ` Andy Lutomirski
2019-11-12 19:40 ` Thomas Gleixner
2019-11-11 22:03 ` [patch V2 16/16] selftests/x86/iopl: Extend test to cover IOPL emulation Thomas Gleixner
2019-11-12 7:40 ` [PATCH] x86/iopl: Factor out IO-bitmap related TSS fields into 'struct x86_io_bitmap' Ingo Molnar
2019-11-12 7:59 ` [PATCH] x86/iopl: Harmonize 'struct io_bitmap' and 'struct x86_io_bitmap' nomenclature Ingo Molnar
2019-11-12 8:11 ` [PATCH] x86/iopl: Clear up the role of the two bitmap copying fields Ingo Molnar
2019-11-12 8:15 ` [PATCH] x86/iopl: Rename <asm/iobitmap.h> to <asm/io_bitmap.h> Ingo Molnar
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191111223052.086299881@linutronix.de \
--to=tglx@linutronix.de \
--cc=hpa@zytor.com \
--cc=jgross@suse.com \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=sean.j.christopherson@intel.com \
--cc=stephen@networkplumber.org \
--cc=torvalds@linuxfoundation.org \
--cc=w@1wt.eu \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).