From: Thomas Gleixner <tglx@linutronix.de>
To: Linus Torvalds <torvalds@linux-foundation.org>
Cc: linux-kernel@vger.kernel.org, x86@kernel.org
Subject: [GIT pull] x86/apic for v5.16-rc1
Date: Mon, 1 Nov 2021 02:16:07 +0100 (CET) [thread overview]
Message-ID: <163572865146.3357115.6271807450024716473.tglx@xen13> (raw)
In-Reply-To: 163572864256.3357115.931779940195622047.tglx@xen13
Linus,
please pull the latest x86/apic branch from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86-apic-2021-11-01
up to: cc95a07fef06: x86/apic: Reduce cache line misses in __x2apic_send_IPI_mask()
x86/apic related update:
- A single commit which reduces cacheline misses in
__x2apic_send_IPI_mask() significantly by converting
x86_cpu_to_logical_apicid() to an array instead of using per CPU
storage. This reduces the cost for a full broadcast on a dual socket
system with 256 CPUs from 33 down to 11 microseconds.
Thanks,
tglx
------------------>
Eric Dumazet (1):
x86/apic: Reduce cache line misses in __x2apic_send_IPI_mask()
arch/x86/kernel/apic/x2apic_cluster.c | 27 +++++++++++++++++++++------
1 file changed, 21 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index f4da9bb69a88..e696e22d0531 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -15,9 +15,15 @@ struct cluster_mask {
struct cpumask mask;
};
-static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
+/*
+ * __x2apic_send_IPI_mask() possibly needs to read
+ * x86_cpu_to_logical_apicid for all online cpus in a sequential way.
+ * Using per cpu variable would cost one cache line per cpu.
+ */
+static u32 *x86_cpu_to_logical_apicid __read_mostly;
+
static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
-static DEFINE_PER_CPU(struct cluster_mask *, cluster_masks);
+static DEFINE_PER_CPU_READ_MOSTLY(struct cluster_mask *, cluster_masks);
static struct cluster_mask *cluster_hotplug_mask;
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
@@ -27,7 +33,7 @@ static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
static void x2apic_send_IPI(int cpu, int vector)
{
- u32 dest = per_cpu(x86_cpu_to_logical_apicid, cpu);
+ u32 dest = x86_cpu_to_logical_apicid[cpu];
/* x2apic MSRs are special and need a special fence: */
weak_wrmsr_fence();
@@ -58,7 +64,7 @@ __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
dest = 0;
for_each_cpu_and(clustercpu, tmpmsk, &cmsk->mask)
- dest |= per_cpu(x86_cpu_to_logical_apicid, clustercpu);
+ dest |= x86_cpu_to_logical_apicid[clustercpu];
if (!dest)
continue;
@@ -94,7 +100,7 @@ static void x2apic_send_IPI_all(int vector)
static u32 x2apic_calc_apicid(unsigned int cpu)
{
- return per_cpu(x86_cpu_to_logical_apicid, cpu);
+ return x86_cpu_to_logical_apicid[cpu];
}
static void init_x2apic_ldr(void)
@@ -103,7 +109,7 @@ static void init_x2apic_ldr(void)
u32 cluster, apicid = apic_read(APIC_LDR);
unsigned int cpu;
- this_cpu_write(x86_cpu_to_logical_apicid, apicid);
+ x86_cpu_to_logical_apicid[smp_processor_id()] = apicid;
if (cmsk)
goto update;
@@ -166,12 +172,21 @@ static int x2apic_dead_cpu(unsigned int dead_cpu)
static int x2apic_cluster_probe(void)
{
+ u32 slots;
+
if (!x2apic_mode)
return 0;
+ slots = max_t(u32, L1_CACHE_BYTES/sizeof(u32), nr_cpu_ids);
+ x86_cpu_to_logical_apicid = kcalloc(slots, sizeof(u32), GFP_KERNEL);
+ if (!x86_cpu_to_logical_apicid)
+ return 0;
+
if (cpuhp_setup_state(CPUHP_X2APIC_PREPARE, "x86/x2apic:prepare",
x2apic_prepare_cpu, x2apic_dead_cpu) < 0) {
pr_err("Failed to register X2APIC_PREPARE\n");
+ kfree(x86_cpu_to_logical_apicid);
+ x86_cpu_to_logical_apicid = NULL;
return 0;
}
init_x2apic_ldr();
next prev parent reply other threads:[~2021-11-01 1:16 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-01 1:15 [GIT pull] irq/core for v5.16-rc1 Thomas Gleixner
2021-11-01 1:15 ` [GIT pull] locking/core " Thomas Gleixner
2021-11-01 21:20 ` pr-tracker-bot
2021-11-01 1:16 ` [GIT pull] objtool/core " Thomas Gleixner
2021-11-01 20:44 ` Linus Torvalds
2021-11-02 8:00 ` Peter Zijlstra
2021-11-02 8:06 ` Borislav Petkov
2021-11-02 9:05 ` Stackleak vs noinstr (Was: [GIT pull] objtool/core for v5.16-rc1) Peter Zijlstra
2021-11-02 10:03 ` Peter Zijlstra
2021-11-02 17:50 ` Linus Torvalds
2021-11-02 21:18 ` Borislav Petkov
2021-11-03 7:18 ` Alexander Popov
2021-11-03 8:19 ` Peter Zijlstra
2022-02-01 23:59 ` Kees Cook
2021-11-01 21:20 ` [GIT pull] objtool/core for v5.16-rc1 pr-tracker-bot
2021-11-01 1:16 ` [GIT pull] perf/core " Thomas Gleixner
2021-11-01 21:20 ` pr-tracker-bot
2021-11-01 1:16 ` [GIT pull] sched/core " Thomas Gleixner
2021-11-01 21:01 ` Linus Torvalds
2021-11-01 21:27 ` Linus Torvalds
2021-11-02 8:41 ` Peter Zijlstra
2021-11-03 13:52 ` Mark Rutland
2021-11-03 16:23 ` Linus Torvalds
2021-11-02 8:54 ` Peter Zijlstra
2021-11-01 21:20 ` pr-tracker-bot
2021-11-01 1:16 ` [GIT pull] timers/core " Thomas Gleixner
2021-11-01 21:20 ` pr-tracker-bot
2021-11-01 1:16 ` Thomas Gleixner [this message]
2021-11-01 21:20 ` [GIT pull] x86/apic " pr-tracker-bot
2021-11-01 1:52 ` [GIT pull RESEND] x86/fpu " Thomas Gleixner
2021-11-01 21:20 ` pr-tracker-bot
2021-11-01 21:19 ` [GIT pull] irq/core " pr-tracker-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=163572865146.3357115.6271807450024716473.tglx@xen13 \
--to=tglx@linutronix.de \
--cc=linux-kernel@vger.kernel.org \
--cc=torvalds@linux-foundation.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).