KVM Archive on lore.kernel.org
 help / color / Atom feed
From: David Woodhouse <dwmw2@infradead.org>
To: x86@kernel.org
Cc: kvm <kvm@vger.kernel.org>, Thomas Gleixner <tglx@linutronix.de>,
	Paolo Bonzini <pbonzini@redhat.com>,
	linux-kernel <linux-kernel@vger.kernel.org>,
	linux-hyperv@vger.kernel.org
Subject: [PATCH v2 5/8] x86/apic: Support 15 bits of APIC ID in MSI where available
Date: Fri,  9 Oct 2020 11:46:13 +0100
Message-ID: <20201009104616.1314746-6-dwmw2@infradead.org> (raw)
In-Reply-To: <20201009104616.1314746-1-dwmw2@infradead.org>

From: David Woodhouse <dwmw@amazon.co.uk>

Some hypervisors can allow the guest to use the Extended Destination ID
field in the MSI address to address up to 32768 CPUs.

This applies to all downstream devices which generate MSI cycles,
including HPET, I/OAPIC and PCI MSI.

HPET and PCI MSI use the same __irq_msi_compose_msg() function, while
I/OAPIC generates its own and had support for the extended bits added in
a previous commit.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/include/asm/x86_init.h |  2 ++
 arch/x86/kernel/apic/apic.c     | 26 ++++++++++++++++++++++++--
 arch/x86/kernel/x86_init.c      |  1 +
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index 397196fae24d..96a719fb2e53 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -114,6 +114,7 @@ struct x86_init_pci {
  * @init_platform:		platform setup
  * @guest_late_init:		guest late init
  * @x2apic_available:		X2APIC detection
+ * @msi_ext_dest_id:		MSI supports 15-bit APIC IDs
  * @init_mem_mapping:		setup early mappings during init_mem_mapping()
  * @init_after_bootmem:		guest init after boot allocator is finished
  */
@@ -121,6 +122,7 @@ struct x86_hyper_init {
 	void (*init_platform)(void);
 	void (*guest_late_init)(void);
 	bool (*x2apic_available)(void);
+	bool (*msi_ext_dest_id)(void);
 	void (*init_mem_mapping)(void);
 	void (*init_after_bootmem)(void);
 };
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index fba3ba383ad2..62e769c267c3 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -94,6 +94,11 @@ static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID;
  */
 static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
 
+/*
+ * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
+ */
+static bool virt_ext_dest_id __ro_after_init;
+
 /*
  * Map cpu index to physical APIC ID
  */
@@ -1842,6 +1847,8 @@ static __init void try_to_enable_x2apic(int remap_mode)
 		return;
 
 	if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
+		u32 apic_limit = 255;
+
 		/*
 		 * Using X2APIC without IR is not architecturally supported
 		 * on bare metal but may be supported in guests.
@@ -1852,12 +1859,22 @@ static __init void try_to_enable_x2apic(int remap_mode)
 			return;
 		}
 
+		/*
+		 * If the hypervisor supports extended destination ID in
+		 * MSI, that increases the maximum APIC ID that can be
+		 * used for non-remapped IRQ domains.
+		 */
+		if (x86_init.hyper.msi_ext_dest_id()) {
+			virt_ext_dest_id = 1;
+			apic_limit = 32767;
+		}
+
 		/*
 		 * Without IR, all CPUs can be addressed by IOAPIC/MSI only
 		 * in physical mode, and CPUs with an APIC ID that cannnot
 		 * be addressed must not be brought online.
 		 */
-		x2apic_set_max_apicid(255);
+		x2apic_set_max_apicid(apic_limit);
 		x2apic_phys = 1;
 	}
 	x2apic_enable();
@@ -2504,10 +2521,15 @@ void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
 	 * Only the IOMMU itself can use the trick of putting destination
 	 * APIC ID into the high bits of the address. Anything else would
 	 * just be writing to memory if it tried that, and needs IR to
-	 * address higher APIC IDs.
+	 * address APICs which can't be addressed in the normal 32-bit
+	 * address range at 0xFFExxxxx. That is typically just 8 bits, but
+	 * some hypervisors allow the extended destination ID field in bits
+	 * 11-5 to be used, giving support for 15 bits of APIC IDs in total.
 	 */
 	if (dmar)
 		msg->address_hi |= MSI_ADDR_EXT_DEST_ID(cfg->dest_apicid);
+	else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
+		msg->address_lo |= MSI_ADDR_EXT_DEST_ID(cfg->dest_apicid) >> 3;
 	else
 		WARN_ON_ONCE(MSI_ADDR_EXT_DEST_ID(cfg->dest_apicid));
 }
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
index a3038d8deb6a..8b395821cb8d 100644
--- a/arch/x86/kernel/x86_init.c
+++ b/arch/x86/kernel/x86_init.c
@@ -110,6 +110,7 @@ struct x86_init_ops x86_init __initdata = {
 		.init_platform		= x86_init_noop,
 		.guest_late_init	= x86_init_noop,
 		.x2apic_available	= bool_x86_init_noop,
+		.msi_ext_dest_id	= bool_x86_init_noop,
 		.init_mem_mapping	= x86_init_noop,
 		.init_after_bootmem	= x86_init_noop,
 	},
-- 
2.26.2


  parent reply index

Thread overview: 113+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-10-07 12:20 [PATCH 0/5] Fix x2apic enablement and allow up to 32768 CPUs without IR where supported David Woodhouse
2020-10-07 12:20 ` [PATCH 1/5] x86/apic: Fix x2apic enablement without interrupt remapping David Woodhouse
2020-10-07 12:20   ` [PATCH 2/5] x86/msi: Only use high bits of MSI address for DMAR unit David Woodhouse
2020-10-07 12:20   ` [PATCH 3/5] x86/ioapic: Handle Extended Destination ID field in RTE David Woodhouse
2020-10-08  9:12     ` Peter Zijlstra
2020-10-08 17:05       ` David Woodhouse
2020-10-08 11:41     ` Thomas Gleixner
2020-10-07 12:20   ` [PATCH 4/5] x86/apic: Support 15 bits of APIC ID in IOAPIC/MSI where available David Woodhouse
2020-10-08 11:54     ` Thomas Gleixner
2020-10-08 12:02       ` Thomas Gleixner
2020-10-08 13:00       ` David Woodhouse
2020-10-07 12:20   ` [PATCH 5/5] x86/kvm: Add KVM_FEATURE_MSI_EXT_DEST_ID David Woodhouse
2020-10-08 12:05     ` Thomas Gleixner
2020-10-08 12:55       ` David Woodhouse
2020-10-08 16:08         ` David Woodhouse
2020-10-08 21:14           ` Thomas Gleixner
2020-10-08 21:39             ` David Woodhouse
2020-10-08 23:27               ` Thomas Gleixner
2020-10-09  6:07                 ` David Woodhouse
2020-10-10 10:06                 ` David Woodhouse
2020-10-10 11:44                   ` Thomas Gleixner
2020-10-10 11:58                     ` David Woodhouse
2020-10-11 17:12                       ` Thomas Gleixner
2020-10-11 21:15                         ` David Woodhouse
2020-10-12  9:33                           ` Thomas Gleixner
2020-10-12 16:06                             ` David Woodhouse
2020-10-12 18:38                               ` Thomas Gleixner
2020-10-12 20:20                                 ` David Woodhouse
2020-10-12 22:13                                   ` Thomas Gleixner
2020-10-13  7:52                                     ` David Woodhouse
2020-10-13  8:11                                       ` [PATCH 0/9] Remove irq_remapping_get_irq_domain() David Woodhouse
2020-10-13  8:11                                         ` [PATCH 1/9] genirq/irqdomain: Implement get_name() method on irqchip fwnodes David Woodhouse
2020-10-13  8:11                                         ` [PATCH 2/9] x86/apic: Add select() method on vector irqdomain David Woodhouse
2020-10-13  8:11                                         ` [PATCH 3/9] iommu/amd: Implement select() method on remapping irqdomain David Woodhouse
2020-10-13  8:11                                         ` [PATCH 4/9] iommu/vt-d: " David Woodhouse
2020-10-13  8:11                                         ` [PATCH 5/9] iommu/hyper-v: " David Woodhouse
2020-10-13  8:11                                         ` [PATCH 6/9] x86/hpet: Use irq_find_matching_fwspec() to find " David Woodhouse
2020-10-13  8:11                                         ` [PATCH 7/9] x86/ioapic: " David Woodhouse
2020-10-13  8:11                                         ` [PATCH 8/9] x86: Kill all traces of irq_remapping_get_irq_domain() David Woodhouse
2020-10-13  8:11                                         ` [PATCH 9/9] iommu/vt-d: Simplify intel_irq_remapping_select() David Woodhouse
2020-10-13  9:28                                       ` [PATCH 5/5] x86/kvm: Add KVM_FEATURE_MSI_EXT_DEST_ID Thomas Gleixner
2020-10-13 10:15                                         ` David Woodhouse
2020-10-13 10:46                                         ` Thomas Gleixner
2020-10-13 10:53                                           ` David Woodhouse
2020-10-13 11:51                                             ` David Woodhouse
2020-10-13 12:40                                               ` Thomas Gleixner
2020-10-08 11:46   ` [PATCH 1/5] x86/apic: Fix x2apic enablement without interrupt remapping Thomas Gleixner
2020-10-09 10:46 ` [PATCH v2 0/8] Fix x2apic enablement and allow up to 32768 CPUs without IR where supported David Woodhouse
2020-10-09 10:46   ` [PATCH v2 1/8] x86/apic: Fix x2apic enablement without interrupt remapping David Woodhouse
2020-10-09 10:46   ` [PATCH v2 2/8] x86/msi: Only use high bits of MSI address for DMAR unit David Woodhouse
2020-10-09 10:46   ` [PATCH v2 3/8] x86/apic: Always provide irq_compose_msi_msg() method for vector domain David Woodhouse
2020-10-09 10:46   ` [PATCH v2 4/8] x86/ioapic: Handle Extended Destination ID field in RTE David Woodhouse
2020-10-09 10:46   ` David Woodhouse [this message]
2020-10-09 10:46   ` [PATCH v2 6/8] x86/kvm: Add KVM_FEATURE_MSI_EXT_DEST_ID David Woodhouse
2020-10-09 10:46   ` [PATCH v2 7/8] x86/hpet: Move MSI support into hpet.c David Woodhouse
2020-10-09 10:46   ` [PATCH v2 8/8] x86/ioapic: Generate RTE directly from parent irqchip's MSI message David Woodhouse
2020-10-22 21:43     ` Thomas Gleixner
2020-10-22 22:10       ` Thomas Gleixner
2020-10-23 17:04         ` David Woodhouse
2020-10-23 10:10       ` David Woodhouse
2020-10-23 21:28         ` Thomas Gleixner
2020-10-24  8:26           ` David Woodhouse
2020-10-24  8:41             ` David Woodhouse
2020-10-24  9:13             ` Paolo Bonzini
2020-10-24 10:13               ` David Woodhouse
2020-10-24 12:44                 ` David Woodhouse
2020-10-24 21:35                   ` [PATCH v3 00/35] Fix x2apic enablement and allow more CPUs, clean up I/OAPIC and MSI bitfields David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 01/35] x86/apic: Fix x2apic enablement without interrupt remapping David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 02/35] x86/msi: Only use high bits of MSI address for DMAR unit David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 03/35] x86/apic/uv: Fix inconsistent destination mode David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 04/35] x86/devicetree: Fix the ioapic interrupt type table David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 05/35] x86/apic: Cleanup delivery mode defines David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 06/35] x86/apic: Replace pointless apic::dest_logical usage David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 07/35] x86/apic: Get rid of apic::dest_logical David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 08/35] x86/apic: Cleanup destination mode David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 09/35] x86/apic: Always provide irq_compose_msi_msg() method for vector domain David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 10/35] x86/hpet: Move MSI support into hpet.c David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 11/35] genirq/msi: Allow shadow declarations of msi_msg::$member David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 12/35] x86/msi: Provide msi message shadow structs David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 13/35] iommu/intel: Use msi_msg " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 14/35] iommu/amd: " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 15/35] PCI: vmd: " David Woodhouse
2020-10-28 20:49                       ` Kees Cook
2020-10-28 21:13                         ` Thomas Gleixner
2020-10-28 23:22                           ` Kees Cook
2020-10-24 21:35                     ` [PATCH v3 16/35] x86/kvm: " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 17/35] x86/pci/xen: " David Woodhouse
2020-10-25  9:49                       ` David Laight
2020-10-25 10:26                         ` David Woodhouse
2020-10-25 13:20                           ` David Laight
2020-10-24 21:35                     ` [PATCH v3 18/35] x86/msi: Remove msidef.h David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 19/35] x86/io_apic: Cleanup trigger/polarity helpers David Woodhouse
2020-11-10  6:31                       ` Qian Cai
2020-11-10  8:59                         ` David Woodhouse
2020-11-10 16:26                           ` Paolo Bonzini
2020-10-24 21:35                     ` [PATCH v3 20/35] x86/ioapic: Cleanup IO/APIC route entry structs David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 21/35] x86/ioapic: Generate RTE directly from parent irqchip's MSI message David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 22/35] genirq/irqdomain: Implement get_name() method on irqchip fwnodes David Woodhouse
2020-10-25  9:41                       ` Marc Zyngier
2020-10-24 21:35                     ` [PATCH v3 23/35] x86/apic: Add select() method on vector irqdomain David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 24/35] iommu/amd: Implement select() method on remapping irqdomain David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 25/35] iommu/vt-d: " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 26/35] iommu/hyper-v: " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 27/35] x86/hpet: Use irq_find_matching_fwspec() to find " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 28/35] x86/ioapic: " David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 29/35] x86: Kill all traces of irq_remapping_get_irq_domain() David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 30/35] iommu/vt-d: Simplify intel_irq_remapping_select() David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 31/35] x86/ioapic: Handle Extended Destination ID field in RTE David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 32/35] x86/apic: Support 15 bits of APIC ID in MSI where available David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 33/35] iommu/hyper-v: Disable IRQ pseudo-remapping if 15 bit APIC IDs are available David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 34/35] x86/kvm: Reserve KVM_FEATURE_MSI_EXT_DEST_ID David Woodhouse
2020-10-24 21:35                     ` [PATCH v3 35/35] x86/kvm: Enable 15-bit extension when KVM_FEATURE_MSI_EXT_DEST_ID detected David Woodhouse
2020-10-25  8:12                     ` [PATCH v3 00/35] Fix x2apic enablement and allow more CPUs, clean up I/OAPIC and MSI bitfields David Woodhouse

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20201009104616.1314746-6-dwmw2@infradead.org \
    --to=dwmw2@infradead.org \
    --cc=kvm@vger.kernel.org \
    --cc=linux-hyperv@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=pbonzini@redhat.com \
    --cc=tglx@linutronix.de \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

KVM Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/kvm/0 kvm/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 kvm kvm/ https://lore.kernel.org/kvm \
		kvm@vger.kernel.org
	public-inbox-index kvm

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.kvm


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git