linux-pci.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Jon Derrick <jonathan.derrick@intel.com>
To: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: <linux-pci@vger.kernel.org>, Bjorn Helgaas <helgaas@kernel.org>,
	Nirmal Patel <nirmal.patel@intel.com>,
	Sushma Kalakota <sushmax.kalakota@intel.com>,
	Jon Derrick <jonathan.derrick@intel.com>
Subject: [PATCH 5/5] PCI: vmd: Add legacy guest passthrough mode
Date: Fri, 20 Nov 2020 15:51:44 -0700	[thread overview]
Message-ID: <20201120225144.15138-6-jonathan.derrick@intel.com> (raw)
In-Reply-To: <20201120225144.15138-1-jonathan.derrick@intel.com>

Some hypervisors allow passthrough of VMD to guests, but don't supply
the emulated vendor-specific capability. VMD users currently
passing-through VMD rely on a preconfiguration of the VMD Root Ports to
inform the guest of the physical addresses for offset mapping the bridge
windows.

This patch adds a non-visible module parameter to activate host or guest
passthrough mode. In host mode, this patch will write out the VMD MEMBAR
information into the root ports on module unload. Guest mode will use
the direct-assign hints, saving the host-supplied root port information
on VMD module load and restore on exit. It uses this information in the
offset calculation for bridge windows.

This is enabled by non-visible module parameter because it is
non-standard use case for certain users for a legacy behavior.

Link: https://lore.kernel.org/linux-pci/20200706091625.GA26377@e121166-lin.cambridge.arm.com/
Signed-off-by: Sushma Kalakota <sushmax.kalakota@intel.com>
Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 drivers/pci/controller/vmd.c | 127 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 126 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index 71aa002..711bbee 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -35,6 +35,19 @@
 #define MB2_SHADOW_OFFSET	0x2000
 #define MB2_SHADOW_SIZE		16
 
+enum legacy_da_mode {
+	VMD_DA_NONE,
+	VMD_DA_HOST,
+	VMD_DA_GUEST,
+};
+
+static int legacy_da_mode;
+static char legacy_da_mode_str[sizeof("guest")];
+module_param_string(legacy_da_mode, legacy_da_mode_str,
+		    sizeof(legacy_da_mode_str), 0);
+MODULE_PARM_DESC(legacy_da_mode,
+	"use legacy host-provided addressing hints in Root Ports to assist guest passthrough (off, host, guest)");
+
 enum vmd_features {
 	/*
 	 * Device may contain registers which hint the physical location of the
@@ -97,6 +110,12 @@ struct vmd_irq_list {
 	unsigned int		count;
 };
 
+struct root_port_addr {
+	int port;
+	u64 membase;
+	u64 pref_membase;
+};
+
 struct vmd_dev {
 	struct pci_dev		*dev;
 
@@ -112,6 +131,7 @@ struct vmd_dev {
 	struct pci_bus		*bus;
 	u8			busn_start;
 	u8			first_vec;
+	struct root_port_addr	rp_addr;
 };
 
 static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
@@ -483,6 +503,97 @@ static int vmd_find_free_domain(void)
 	return domain + 1;
 }
 
+#define VMD_RP_BASE(vmd, port) ((vmd)->cfgbar + (port) * 8 * 4096)
+static void vmd_save_root_port_info(struct vmd_dev *vmd)
+{
+	resource_size_t physical = 0;
+	char __iomem *addr;
+	int port;
+
+	if (upper_32_bits(pci_resource_start(vmd->dev, VMD_MEMBAR1)))
+		return;
+
+	for (port = 0; port < 4; port++) {
+		u32 membase;
+
+		addr = VMD_RP_BASE(vmd, port) + PCI_MEMORY_BASE;
+		membase = readl(addr);
+
+		/* Break on first found root port */
+		if ((membase != 0xffffffff) && (membase != 0) &&
+		    (membase != 0x0000fff0))
+			break;
+	}
+
+	if (port >= 4)
+		return;
+
+	vmd->rp_addr.port = port;
+
+	/* Only save the first root port index in host mode */
+	if (legacy_da_mode == VMD_DA_HOST)
+		return;
+
+	addr = VMD_RP_BASE(vmd, port) + PCI_MEMORY_BASE;
+	physical = ((u64)readw(addr) & 0xfff0) << 16;
+	vmd->rp_addr.membase = physical;
+
+	addr = VMD_RP_BASE(vmd, port) + PCI_PREF_BASE_UPPER32;
+	physical = ((u64)readl(addr)) << 32;
+	vmd->rp_addr.pref_membase = physical;
+
+	addr = VMD_RP_BASE(vmd, port) + PCI_PREF_MEMORY_BASE;
+	physical |= ((u64)readw(addr) & 0xfff0) << 16;
+	vmd->rp_addr.pref_membase |= physical;
+
+	writel(0, VMD_RP_BASE(vmd, port) + PCI_MEMORY_BASE);
+	writel(0, VMD_RP_BASE(vmd, port) + PCI_PREF_BASE_UPPER32);
+	writel(0, VMD_RP_BASE(vmd, port) + PCI_PREF_MEMORY_BASE);
+	writel(0, VMD_RP_BASE(vmd, port) + PCI_PREF_MEMORY_LIMIT);
+}
+
+static void vmd_restore_root_port_info(struct vmd_dev *vmd)
+{
+	resource_size_t	phyaddr;
+	char __iomem *addr;
+	u32 val;
+	int port;
+
+	port = vmd->rp_addr.port;
+	if (legacy_da_mode == VMD_DA_HOST) {
+		/* Write the MEMBAR information to prepare the guest */
+		phyaddr = pci_resource_start(vmd->dev, VMD_MEMBAR1);
+		if (upper_32_bits(phyaddr))
+			return;
+
+		addr = VMD_RP_BASE(vmd, port) + PCI_MEMORY_BASE;
+		val = (phyaddr >> 16) & 0xfff0;
+		writew(val, addr);
+
+		phyaddr = pci_resource_start(vmd->dev, VMD_MEMBAR2);
+		addr = VMD_RP_BASE(vmd, port) + PCI_PREF_BASE_UPPER32;
+		val = phyaddr >> 32;
+		writel(val, addr);
+
+		addr = VMD_RP_BASE(vmd, port) + PCI_PREF_MEMORY_BASE;
+		val = (phyaddr >> 16) & 0xfff0;
+		writew(val, addr);
+	} else if (legacy_da_mode == VMD_DA_GUEST) {
+		/* Restore information provided by Host */
+		addr = VMD_RP_BASE(vmd, port) + PCI_MEMORY_BASE;
+		val = (vmd->rp_addr.membase >> 16) & 0xfff0;
+		writew(val, addr);
+
+		addr = VMD_RP_BASE(vmd, port) + PCI_PREF_BASE_UPPER32;
+		val = vmd->rp_addr.pref_membase >> 32;
+		writel(val, addr);
+
+		addr = VMD_RP_BASE(vmd, port) + PCI_PREF_MEMORY_BASE;
+		val = (vmd->rp_addr.pref_membase >> 16) & 0xfff0;
+		writew(val, addr);
+	}
+}
+
 static void vmd_phys_to_offset(struct vmd_dev *vmd, u64 phys1, u64 phys2,
 				 resource_size_t *offset1,
 				 resource_size_t *offset2)
@@ -500,7 +611,19 @@ static int vmd_get_phys_offsets(struct vmd_dev *vmd, unsigned long features,
 	struct pci_dev *dev = vmd->dev;
 	u64 phys1, phys2;
 
-	if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {
+	if (!strncmp(legacy_da_mode_str, "host", 4))
+		legacy_da_mode = VMD_DA_HOST;
+	else if (!strncmp(legacy_da_mode_str, "guest", 5))
+		legacy_da_mode = VMD_DA_GUEST;
+
+	if (legacy_da_mode != VMD_DA_NONE) {
+		vmd_save_root_port_info(vmd);
+		if (legacy_da_mode == VMD_DA_GUEST) {
+			vmd_phys_to_offset(vmd, vmd->rp_addr.membase,
+					   vmd->rp_addr.pref_membase,
+					   offset1, offset2);
+		}
+	} else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {
 		u32 vmlock;
 		int ret;
 
@@ -732,6 +855,7 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
 	if (!vmd->bus) {
 		pci_free_resource_list(&resources);
 		vmd_remove_irq_domain(vmd);
+		vmd_restore_root_port_info(vmd);
 		return -ENODEV;
 	}
 
@@ -821,6 +945,7 @@ static void vmd_remove(struct pci_dev *dev)
 	vmd_cleanup_srcu(vmd);
 	vmd_detach_resources(vmd);
 	vmd_remove_irq_domain(vmd);
+	vmd_restore_root_port_info(vmd);
 }
 
 #ifdef CONFIG_PM_SLEEP
-- 
1.8.3.1


  parent reply	other threads:[~2020-11-20 22:52 UTC|newest]

Thread overview: 13+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-11-20 22:51 [PATCH 0/5] Legacy direct-assign mode Jon Derrick
2020-11-20 22:51 ` [PATCH 1/5] PCI: vmd: Reset the VMD subdevice domain on probe Jon Derrick
2020-11-20 22:51 ` [PATCH 2/5] PCI: Add a reset quirk for VMD Jon Derrick
2020-11-24 21:40   ` Bjorn Helgaas
2020-11-25 17:22     ` Derrick, Jonathan
2020-11-25 17:34       ` Alex Williamson
2020-11-20 22:51 ` [PATCH 3/5] PCI: vmd: Add offset translation helper Jon Derrick
2020-11-20 22:51 ` [PATCH 4/5] PCI: vmd: Pass features to vmd_get_phys_offsets() Jon Derrick
2020-11-20 22:51 ` Jon Derrick [this message]
2021-03-22 12:28 ` [PATCH 0/5] Legacy direct-assign mode Lorenzo Pieralisi
2021-03-22 15:25   ` Derrick, Jonathan
2021-03-22 19:48   ` Christoph Hellwig
2021-03-22 22:55     ` Derrick, Jonathan

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20201120225144.15138-6-jonathan.derrick@intel.com \
    --to=jonathan.derrick@intel.com \
    --cc=helgaas@kernel.org \
    --cc=linux-pci@vger.kernel.org \
    --cc=lorenzo.pieralisi@arm.com \
    --cc=nirmal.patel@intel.com \
    --cc=sushmax.kalakota@intel.com \
    --subject='Re: [PATCH 5/5] PCI: vmd: Add legacy guest passthrough mode' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).