From: Valentin Schneider <vschneid@redhat.com>
To: linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org,
linux-doc@vger.kernel.org, kvm@vger.kernel.org,
linux-mm@kvack.org, bpf@vger.kernel.org, x86@kernel.org,
rcu@vger.kernel.org, linux-kselftest@vger.kernel.org
Cc: "Steven Rostedt" <rostedt@goodmis.org>,
"Masami Hiramatsu" <mhiramat@kernel.org>,
"Jonathan Corbet" <corbet@lwn.net>,
"Thomas Gleixner" <tglx@linutronix.de>,
"Ingo Molnar" <mingo@redhat.com>,
"Borislav Petkov" <bp@alien8.de>,
"Dave Hansen" <dave.hansen@linux.intel.com>,
"H. Peter Anvin" <hpa@zytor.com>,
"Paolo Bonzini" <pbonzini@redhat.com>,
"Wanpeng Li" <wanpengli@tencent.com>,
"Vitaly Kuznetsov" <vkuznets@redhat.com>,
"Andy Lutomirski" <luto@kernel.org>,
"Peter Zijlstra" <peterz@infradead.org>,
"Frederic Weisbecker" <frederic@kernel.org>,
"Paul E. McKenney" <paulmck@kernel.org>,
"Neeraj Upadhyay" <quic_neeraju@quicinc.com>,
"Joel Fernandes" <joel@joelfernandes.org>,
"Josh Triplett" <josh@joshtriplett.org>,
"Boqun Feng" <boqun.feng@gmail.com>,
"Mathieu Desnoyers" <mathieu.desnoyers@efficios.com>,
"Lai Jiangshan" <jiangshanlai@gmail.com>,
Zqiang <qiang.zhang1211@gmail.com>,
"Andrew Morton" <akpm@linux-foundation.org>,
"Uladzislau Rezki" <urezki@gmail.com>,
"Christoph Hellwig" <hch@infradead.org>,
"Lorenzo Stoakes" <lstoakes@gmail.com>,
"Josh Poimboeuf" <jpoimboe@kernel.org>,
"Jason Baron" <jbaron@akamai.com>,
"Kees Cook" <keescook@chromium.org>,
"Sami Tolvanen" <samitolvanen@google.com>,
"Ard Biesheuvel" <ardb@kernel.org>,
"Nicholas Piggin" <npiggin@gmail.com>,
"Juerg Haefliger" <juerg.haefliger@canonical.com>,
"Nicolas Saenz Julienne" <nsaenz@kernel.org>,
"Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>,
"Nadav Amit" <namit@vmware.com>,
"Dan Carpenter" <error27@gmail.com>,
"Chuang Wang" <nashuiliang@gmail.com>,
"Yang Jihong" <yangjihong1@huawei.com>,
"Petr Mladek" <pmladek@suse.com>,
"Jason A. Donenfeld" <Jason@zx2c4.com>,
"Song Liu" <song@kernel.org>,
"Julian Pidancet" <julian.pidancet@oracle.com>,
"Tom Lendacky" <thomas.lendacky@amd.com>,
"Dionna Glaze" <dionnaglaze@google.com>,
"Thomas Weißschuh" <linux@weissschuh.net>,
"Juri Lelli" <juri.lelli@redhat.com>,
"Daniel Bristot de Oliveira" <bristot@redhat.com>,
"Marcelo Tosatti" <mtosatti@redhat.com>,
"Yair Podemsky" <ypodemsk@redhat.com>
Subject: [RFC PATCH v2 19/20] context_tracking,x86: Add infrastructure to defer kernel TLBI
Date: Thu, 20 Jul 2023 17:30:55 +0100 [thread overview]
Message-ID: <20230720163056.2564824-20-vschneid@redhat.com> (raw)
In-Reply-To: <20230720163056.2564824-1-vschneid@redhat.com>
Kernel TLB invalidation IPIs are a common source of interference on
NOHZ_FULL CPUs. Given NOHZ_FULL CPUs executing in userspace are not
accessing any kernel addresses, these invalidations do not need to happen
immediately, and can be deferred until the next user->kernel transition.
Rather than make __flush_tlb_all() noinstr, add a minimal noinstr
variant that doesn't try to leverage INVPCID.
FIXME: not fully noinstr compliant
XXX: same issue as with ins patching, when do we access data that should be
invalidated?
Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
arch/x86/include/asm/context_tracking_work.h | 4 ++++
arch/x86/include/asm/tlbflush.h | 1 +
arch/x86/mm/tlb.c | 17 +++++++++++++++++
include/linux/context_tracking_state.h | 4 ++++
include/linux/context_tracking_work.h | 2 ++
5 files changed, 28 insertions(+)
diff --git a/arch/x86/include/asm/context_tracking_work.h b/arch/x86/include/asm/context_tracking_work.h
index 2c66687ce00e2..9d4f021b5a45b 100644
--- a/arch/x86/include/asm/context_tracking_work.h
+++ b/arch/x86/include/asm/context_tracking_work.h
@@ -3,6 +3,7 @@
#define _ASM_X86_CONTEXT_TRACKING_WORK_H
#include <asm/sync_core.h>
+#include <asm/tlbflush.h>
static __always_inline void arch_context_tracking_work(int work)
{
@@ -10,6 +11,9 @@ static __always_inline void arch_context_tracking_work(int work)
case CONTEXT_WORK_SYNC:
sync_core();
break;
+ case CONTEXT_WORK_TLBI:
+ __flush_tlb_all_noinstr();
+ break;
}
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 80450e1d5385a..323b971987af7 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -17,6 +17,7 @@
DECLARE_PER_CPU(u64, tlbstate_untag_mask);
void __flush_tlb_all(void);
+void noinstr __flush_tlb_all_noinstr(void);
#define TLB_FLUSH_ALL -1UL
#define TLB_GENERATION_INVALID 0
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 267acf27480af..631df9189ded4 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -1237,6 +1237,23 @@ void __flush_tlb_all(void)
}
EXPORT_SYMBOL_GPL(__flush_tlb_all);
+void noinstr __flush_tlb_all_noinstr(void)
+{
+ /*
+ * This is for invocation in early entry code that cannot be
+ * instrumented. A RMW to CR4 works for most cases, but relies on
+ * being able to flip either of the PGE or PCIDE bits. Flipping CR4.PCID
+ * would require also resetting CR3.PCID, so just try with CR4.PGE, else
+ * do the CR3 write.
+ *
+ * TODO: paravirt
+ */
+ if (cpu_feature_enabled(X86_FEATURE_PGE))
+ __native_tlb_flush_global(this_cpu_read(cpu_tlbstate.cr4));
+ else
+ flush_tlb_local();
+}
+
void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
struct flush_tlb_info *info;
diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h
index 292a0b7c06948..3571c62cbb9cd 100644
--- a/include/linux/context_tracking_state.h
+++ b/include/linux/context_tracking_state.h
@@ -62,6 +62,10 @@ enum ctx_state {
#define RCU_DYNTICKS_END (CT_STATE_SIZE - 1)
#define RCU_DYNTICKS_IDX BIT(RCU_DYNTICKS_START)
+/*
+ * When CONFIG_CONTEXT_TRACKING_WORK=n, _END is 1 behind _START, which makes
+ * the CONTEXT_WORK size computation below 0, which is what we want!
+ */
#define CONTEXT_WORK_START (CONTEXT_STATE_END + 1)
#define CONTEXT_WORK_END (RCU_DYNTICKS_START - 1)
diff --git a/include/linux/context_tracking_work.h b/include/linux/context_tracking_work.h
index 13fc97b395030..47d5ced39a43a 100644
--- a/include/linux/context_tracking_work.h
+++ b/include/linux/context_tracking_work.h
@@ -6,11 +6,13 @@
enum {
CONTEXT_WORK_SYNC_OFFSET,
+ CONTEXT_WORK_TLBI_OFFSET,
CONTEXT_WORK_MAX_OFFSET
};
enum ct_work {
CONTEXT_WORK_SYNC = BIT(CONTEXT_WORK_SYNC_OFFSET),
+ CONTEXT_WORK_TLBI = BIT(CONTEXT_WORK_TLBI_OFFSET),
CONTEXT_WORK_MAX = BIT(CONTEXT_WORK_MAX_OFFSET)
};
--
2.31.1
next prev parent reply other threads:[~2023-07-20 16:34 UTC|newest]
Thread overview: 76+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-07-20 16:30 [RFC PATCH v2 00/20] context_tracking,x86: Defer some IPIs until a user->kernel transition Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 01/20] tracing/filters: Dynamically allocate filter_pred.regex Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 02/20] tracing/filters: Enable filtering a cpumask field by another cpumask Valentin Schneider
2023-07-26 19:41 ` Josh Poimboeuf
2023-07-27 9:46 ` Valentin Schneider
2023-07-29 19:09 ` Steven Rostedt
2023-07-31 11:19 ` Valentin Schneider
2023-07-31 15:48 ` Steven Rostedt
2023-07-20 16:30 ` [RFC PATCH v2 03/20] tracing/filters: Enable filtering a scalar field by a cpumask Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 04/20] tracing/filters: Enable filtering the CPU common " Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 05/20] tracing/filters: Optimise cpumask vs cpumask filtering when user mask is a single CPU Valentin Schneider
2023-07-29 19:34 ` Steven Rostedt
2023-07-31 11:20 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 06/20] tracing/filters: Optimise scalar vs cpumask filtering when the " Valentin Schneider
2023-07-29 19:55 ` Steven Rostedt
2023-07-31 11:20 ` Valentin Schneider
2023-07-31 12:07 ` Dan Carpenter
2023-07-31 15:54 ` Steven Rostedt
2023-07-31 16:03 ` Dan Carpenter
2023-07-31 17:20 ` Valentin Schneider
2023-07-31 18:16 ` Steven Rostedt
2023-07-20 16:30 ` [RFC PATCH v2 07/20] tracing/filters: Optimise CPU " Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 08/20] tracing/filters: Further optimise scalar vs cpumask comparison Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 09/20] tracing/filters: Document cpumask filtering Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 10/20] jump_label,module: Don't alloc static_key_mod for __ro_after_init keys Valentin Schneider
2023-07-28 22:04 ` Peter Zijlstra
2023-07-20 16:30 ` [RFC PATCH v2 11/20] objtool: Flesh out warning related to pv_ops[] calls Valentin Schneider
2023-07-28 15:33 ` Josh Poimboeuf
2023-07-31 11:16 ` Valentin Schneider
2023-07-31 21:36 ` Josh Poimboeuf
2023-07-31 21:46 ` Peter Zijlstra
2023-08-01 16:06 ` Josh Poimboeuf
2023-08-01 18:12 ` Peter Zijlstra
2023-07-20 16:30 ` [RFC PATCH v2 12/20] objtool: Warn about non __ro_after_init static key usage in .noinstr Valentin Schneider
2023-07-28 15:35 ` Josh Poimboeuf
2023-07-31 11:18 ` Valentin Schneider
2023-07-28 16:02 ` Josh Poimboeuf
2023-07-31 11:18 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 13/20] context_tracking: Make context_tracking_key __ro_after_init Valentin Schneider
2023-07-28 16:00 ` Josh Poimboeuf
2023-07-31 11:16 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 14/20] x86/kvm: Make kvm_async_pf_enabled __ro_after_init Valentin Schneider
2023-10-09 16:40 ` Maxim Levitsky
2023-07-20 16:30 ` [RFC PATCH v2 15/20] context-tracking: Introduce work deferral infrastructure Valentin Schneider
2023-07-24 14:52 ` Frederic Weisbecker
2023-07-24 16:55 ` Valentin Schneider
2023-07-24 19:18 ` Frederic Weisbecker
2023-07-25 10:10 ` Valentin Schneider
2023-07-25 11:22 ` Frederic Weisbecker
2023-07-25 13:05 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 16/20] rcu: Make RCU dynticks counter size configurable Valentin Schneider
2023-07-21 8:17 ` Valentin Schneider
2023-07-21 14:10 ` Paul E. McKenney
2023-07-21 15:08 ` Valentin Schneider
2023-07-21 16:09 ` Paul E. McKenney
2023-07-20 16:30 ` [RFC PATCH v2 17/20] rcutorture: Add a test config to torture test low RCU_DYNTICKS width Valentin Schneider
2023-07-20 19:53 ` Paul E. McKenney
2023-07-21 4:00 ` Paul E. McKenney
2023-07-21 7:58 ` Valentin Schneider
2023-07-21 14:07 ` Paul E. McKenney
2023-07-21 15:08 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 18/20] context_tracking,x86: Defer kernel text patching IPIs Valentin Schneider
2023-07-25 10:49 ` Joel Fernandes
2023-07-25 13:36 ` Valentin Schneider
2023-07-25 17:41 ` Joel Fernandes
2023-07-25 13:39 ` Peter Zijlstra
2023-07-25 17:47 ` Joel Fernandes
2023-07-20 16:30 ` Valentin Schneider [this message]
2023-07-20 16:30 ` [RFC PATCH v2 20/20] x86/mm, mm/vmalloc: Defer flush_tlb_kernel_range() targeting NOHZ_FULL CPUs Valentin Schneider
2023-07-21 18:15 ` Nadav Amit
2023-07-24 11:32 ` Valentin Schneider
2023-07-24 17:40 ` Dave Hansen
2023-07-25 13:21 ` Peter Zijlstra
2023-07-25 14:03 ` Valentin Schneider
2023-07-25 16:37 ` Marcelo Tosatti
2023-07-25 17:12 ` Dave Hansen
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230720163056.2564824-20-vschneid@redhat.com \
--to=vschneid@redhat.com \
--cc=Jason@zx2c4.com \
--cc=akpm@linux-foundation.org \
--cc=ardb@kernel.org \
--cc=boqun.feng@gmail.com \
--cc=bp@alien8.de \
--cc=bpf@vger.kernel.org \
--cc=bristot@redhat.com \
--cc=corbet@lwn.net \
--cc=dave.hansen@linux.intel.com \
--cc=dionnaglaze@google.com \
--cc=error27@gmail.com \
--cc=frederic@kernel.org \
--cc=hch@infradead.org \
--cc=hpa@zytor.com \
--cc=jbaron@akamai.com \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=jpoimboe@kernel.org \
--cc=juerg.haefliger@canonical.com \
--cc=julian.pidancet@oracle.com \
--cc=juri.lelli@redhat.com \
--cc=keescook@chromium.org \
--cc=kirill.shutemov@linux.intel.com \
--cc=kvm@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-trace-kernel@vger.kernel.org \
--cc=linux@weissschuh.net \
--cc=lstoakes@gmail.com \
--cc=luto@kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mhiramat@kernel.org \
--cc=mingo@redhat.com \
--cc=mtosatti@redhat.com \
--cc=namit@vmware.com \
--cc=nashuiliang@gmail.com \
--cc=npiggin@gmail.com \
--cc=nsaenz@kernel.org \
--cc=paulmck@kernel.org \
--cc=pbonzini@redhat.com \
--cc=peterz@infradead.org \
--cc=pmladek@suse.com \
--cc=qiang.zhang1211@gmail.com \
--cc=quic_neeraju@quicinc.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=samitolvanen@google.com \
--cc=song@kernel.org \
--cc=tglx@linutronix.de \
--cc=thomas.lendacky@amd.com \
--cc=urezki@gmail.com \
--cc=vkuznets@redhat.com \
--cc=wanpengli@tencent.com \
--cc=x86@kernel.org \
--cc=yangjihong1@huawei.com \
--cc=ypodemsk@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).