detect and warn about ahci-remapped NVMe devices

detect and warn about ahci-remapped NVMe devices

[Christoph Hellwig]

Just a small band-aid to detect the AHCI-remapped NVMe devices
for now.  Hopefully we'll eventually get a fake PCIe root port
driver ala VMD for them.

detect and warn about ahci-remapped NVMe devices

[Christoph Hellwig]

Just a small band-aid to detect the AHCI-remapped NVMe devices
for now.  Hopefully we'll eventually get a fake PCIe root port
driver ala VMD for them.

[PATCH 1/3] nvme: move NVMe class code to pci_ids.h

[Christoph Hellwig]

We'll need to check for it in the AHCI drivers (yes, really) soon.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 drivers/nvme/host/pci.c | 3 ---
 include/linux/pci_ids.h | 2 ++
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0fc99f0..2c58f15 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -2064,9 +2064,6 @@ static const struct pci_error_handlers nvme_err_handler = {
 	.reset_notify	= nvme_reset_notify,
 };
 
-/* Move to pci_ids.h later */
-#define PCI_CLASS_STORAGE_EXPRESS	0x010802
-
 static const struct pci_device_id nvme_id_table[] = {
 	{ PCI_VDEVICE(INTEL, 0x0953),
 		.driver_data = NVME_QUIRK_STRIPE_SIZE |
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index c58752f..a5e6c7b 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -23,8 +23,10 @@
 #define PCI_CLASS_STORAGE_SATA		0x0106
 #define PCI_CLASS_STORAGE_SATA_AHCI	0x010601
 #define PCI_CLASS_STORAGE_SAS		0x0107
+#define PCI_CLASS_STORAGE_EXPRESS	0x010802
 #define PCI_CLASS_STORAGE_OTHER		0x0180
 
+
 #define PCI_BASE_CLASS_NETWORK		0x02
 #define PCI_CLASS_NETWORK_ETHERNET	0x0200
 #define PCI_CLASS_NETWORK_TOKEN_RING	0x0201
-- 
2.1.4

[PATCH 1/3] nvme: move NVMe class code to pci_ids.h

[Christoph Hellwig]

We'll need to check for it in the AHCI drivers (yes, really) soon.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/nvme/host/pci.c | 3 ---
 include/linux/pci_ids.h | 2 ++
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0fc99f0..2c58f15 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -2064,9 +2064,6 @@ static const struct pci_error_handlers nvme_err_handler = {
 	.reset_notify	= nvme_reset_notify,
 };
 
-/* Move to pci_ids.h later */
-#define PCI_CLASS_STORAGE_EXPRESS	0x010802
-
 static const struct pci_device_id nvme_id_table[] = {
 	{ PCI_VDEVICE(INTEL, 0x0953),
 		.driver_data = NVME_QUIRK_STRIPE_SIZE |
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index c58752f..a5e6c7b 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -23,8 +23,10 @@
 #define PCI_CLASS_STORAGE_SATA		0x0106
 #define PCI_CLASS_STORAGE_SATA_AHCI	0x010601
 #define PCI_CLASS_STORAGE_SAS		0x0107
+#define PCI_CLASS_STORAGE_EXPRESS	0x010802
 #define PCI_CLASS_STORAGE_OTHER		0x0180
 
+
 #define PCI_BASE_CLASS_NETWORK		0x02
 #define PCI_CLASS_NETWORK_ETHERNET	0x0200
 #define PCI_CLASS_NETWORK_TOKEN_RING	0x0201
-- 
2.1.4

[PATCH 2/3] ahci-remap.h: add ahci remapping definitions

[Christoph Hellwig]

From: Dan Williams <dan.j.williams@intel.com>

Signed-off-by: Dan Williams <dan.j.williams@intel.com>
[hch: split into a separate header and commit]
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 include/linux/ahci-remap.h

diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
new file mode 100644
index 0000000..883d366
--- /dev/null
+++ b/include/linux/ahci-remap.h
@@ -0,0 +1,29 @@
+#ifndef _LINUX_AHCI_REMAP_H
+#define _LINUX_AHCI_REMAP_H
+
+#include <linux/sizes.h>
+
+#define AHCI_VSCAP		0xa4
+#define AHCI_REMAP_CAP		0x800
+#define AHCI_VSCAP		0xa4
+
+/* device class code */
+#define AHCI_REMAP_N_DCC	0x880
+
+/* remap-device base relative to ahci-bar */
+#define AHCI_REMAP_N_OFFSET	SZ_16K
+#define AHCI_REMAP_N_SIZE	SZ_16K
+
+#define AHCI_MAX_REMAP		3
+
+static inline unsigned int ahci_remap_dcc(int i)
+{
+	return AHCI_REMAP_N_DCC + i * 0x80;
+}
+
+static inline unsigned int ahci_remap_base(int i)
+{
+	return AHCI_REMAP_N_OFFSET + i * AHCI_REMAP_N_SIZE;
+}
+
+#endif /* _LINUX_AHCI_REMAP_H */
-- 
2.1.4

[PATCH 2/3] ahci-remap.h: add ahci remapping definitions

[Christoph Hellwig]

From: Dan Williams <dan.j.williams@intel.com>

Signed-off-by: Dan Williams <dan.j.williams at intel.com>
[hch: split into a separate header and commit]
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 include/linux/ahci-remap.h

diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
new file mode 100644
index 0000000..883d366
--- /dev/null
+++ b/include/linux/ahci-remap.h
@@ -0,0 +1,29 @@
+#ifndef _LINUX_AHCI_REMAP_H
+#define _LINUX_AHCI_REMAP_H
+
+#include <linux/sizes.h>
+
+#define AHCI_VSCAP		0xa4
+#define AHCI_REMAP_CAP		0x800
+#define AHCI_VSCAP		0xa4
+
+/* device class code */
+#define AHCI_REMAP_N_DCC	0x880
+
+/* remap-device base relative to ahci-bar */
+#define AHCI_REMAP_N_OFFSET	SZ_16K
+#define AHCI_REMAP_N_SIZE	SZ_16K
+
+#define AHCI_MAX_REMAP		3
+
+static inline unsigned int ahci_remap_dcc(int i)
+{
+	return AHCI_REMAP_N_DCC + i * 0x80;
+}
+
+static inline unsigned int ahci_remap_base(int i)
+{
+	return AHCI_REMAP_N_OFFSET + i * AHCI_REMAP_N_SIZE;
+}
+
+#endif /* _LINUX_AHCI_REMAP_H */
-- 
2.1.4

[PATCH 3/3] ahci: warn about remapped NVMe devices

[Christoph Hellwig]

Some Intel ahci implementations have a completely broken remapping mode
where they hide one or more NVMe devices behind the bar of an AHCI device.

Intel refuses to let the OS reprogram the BIOS to switch out of this
mode at runtime, and so far we're not come up with another good way
to undo the mess that the Chipset people created.  So for now the only
thing we can do is to alert users about this situation and switch to the
faster and much saner so called "AHCI" mode insted of the RAID mode in
the BIOS so that the BIOS does not hide the NVMe devices from us.

The sitation is even worse as at least one vendor (thanks a lot Lenovo..)
has started hardcoding their BIOS into the "RAID" mode even for laptops
that don't use AHCI _at all_ and just have a single NVMe device.  For now
there is an unspported Linux-only BIOS that undoes this braindamage,
but we'll have to see if things are getting better or worse from here.

Based on an earlier patch from Dan Williams <dan.j.williams@intel.com>.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 drivers/ata/ahci.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index ba5f11c..516a689 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -46,6 +46,8 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
+#include <linux/ahci-remap.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
 #include "ahci.h"
 
 #define DRV_NAME	"ahci"
@@ -1400,6 +1402,40 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
 }
 #endif
 
+static void ahci_remap_check(struct pci_dev *pdev, int bar,
+		struct ahci_host_priv *hpriv)
+{
+	int i, count = 0;
+	u32 cap;
+
+	/*
+	 * Check if this device might have remapped nvme devices.
+	 */
+	if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
+	    pci_resource_len(pdev, bar) < SZ_512K ||
+	    bar != AHCI_PCI_BAR_STANDARD ||
+	    !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
+		return;
+
+	cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
+	for (i = 0; i < AHCI_MAX_REMAP; i++) {
+		if ((cap & (1 << i)) == 0)
+			continue;
+		if (readl(hpriv->mmio + ahci_remap_dcc(i))
+				!= PCI_CLASS_STORAGE_EXPRESS)
+			continue;
+
+		/* We've found a remapped device */
+		count++;
+	}
+
+	if (!count)
+		return;
+
+	dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count);
+	dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n");
+}
+
 static int ahci_get_irq_vector(struct ata_host *host, int port)
 {
 	return pci_irq_vector(to_pci_dev(host->dev), port);
@@ -1545,6 +1581,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
 
+	/* detect remapped nvme devices */
+	ahci_remap_check(pdev, ahci_pci_bar, hpriv);
+
 	/* must set flag prior to save config in order to take effect */
 	if (ahci_broken_devslp(pdev))
 		hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
-- 
2.1.4

[PATCH 3/3] ahci: warn about remapped NVMe devices

[Christoph Hellwig]

Some Intel ahci implementations have a completely broken remapping mode
where they hide one or more NVMe devices behind the bar of an AHCI device.

Intel refuses to let the OS reprogram the BIOS to switch out of this
mode at runtime, and so far we're not come up with another good way
to undo the mess that the Chipset people created.  So for now the only
thing we can do is to alert users about this situation and switch to the
faster and much saner so called "AHCI" mode insted of the RAID mode in
the BIOS so that the BIOS does not hide the NVMe devices from us.

The sitation is even worse as at least one vendor (thanks a lot Lenovo..)
has started hardcoding their BIOS into the "RAID" mode even for laptops
that don't use AHCI _at all_ and just have a single NVMe device.  For now
there is an unspported Linux-only BIOS that undoes this braindamage,
but we'll have to see if things are getting better or worse from here.

Based on an earlier patch from Dan Williams <dan.j.williams at intel.com>.

Signed-off-by: Christoph Hellwig <hch at lst.de>
---
 drivers/ata/ahci.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index ba5f11c..516a689 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -46,6 +46,8 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
+#include <linux/ahci-remap.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
 #include "ahci.h"
 
 #define DRV_NAME	"ahci"
@@ -1400,6 +1402,40 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
 }
 #endif
 
+static void ahci_remap_check(struct pci_dev *pdev, int bar,
+		struct ahci_host_priv *hpriv)
+{
+	int i, count = 0;
+	u32 cap;
+
+	/*
+	 * Check if this device might have remapped nvme devices.
+	 */
+	if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
+	    pci_resource_len(pdev, bar) < SZ_512K ||
+	    bar != AHCI_PCI_BAR_STANDARD ||
+	    !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
+		return;
+
+	cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
+	for (i = 0; i < AHCI_MAX_REMAP; i++) {
+		if ((cap & (1 << i)) == 0)
+			continue;
+		if (readl(hpriv->mmio + ahci_remap_dcc(i))
+				!= PCI_CLASS_STORAGE_EXPRESS)
+			continue;
+
+		/* We've found a remapped device */
+		count++;
+	}
+
+	if (!count)
+		return;
+
+	dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count);
+	dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n");
+}
+
 static int ahci_get_irq_vector(struct ata_host *host, int port)
 {
 	return pci_irq_vector(to_pci_dev(host->dev), port);
@@ -1545,6 +1581,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
 
+	/* detect remapped nvme devices */
+	ahci_remap_check(pdev, ahci_pci_bar, hpriv);
+
 	/* must set flag prior to save config in order to take effect */
 	if (ahci_broken_devslp(pdev))
 		hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
-- 
2.1.4

Re: [PATCH 2/3] ahci-remap.h: add ahci remapping definitions

[Sergei Shtylyov]

Hello.

On 12/2/2016 9:31 PM, Christoph Hellwig wrote:

> From: Dan Williams <dan.j.williams@intel.com>
>
> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
> [hch: split into a separate header and commit]
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
>  include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
>  create mode 100644 include/linux/ahci-remap.h
>
> diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
> new file mode 100644
> index 0000000..883d366
> --- /dev/null
> +++ b/include/linux/ahci-remap.h
> @@ -0,0 +1,29 @@
> +#ifndef _LINUX_AHCI_REMAP_H
> +#define _LINUX_AHCI_REMAP_H
> +
> +#include <linux/sizes.h>
> +
> +#define AHCI_VSCAP		0xa4
> +#define AHCI_REMAP_CAP		0x800
> +#define AHCI_VSCAP		0xa4

    Why do it twice?

MBR, Sergei

[PATCH 2/3] ahci-remap.h: add ahci remapping definitions

[Sergei Shtylyov]

Hello.

On 12/2/2016 9:31 PM, Christoph Hellwig wrote:

> From: Dan Williams <dan.j.williams at intel.com>
>
> Signed-off-by: Dan Williams <dan.j.williams at intel.com>
> [hch: split into a separate header and commit]
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
>  include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
>  create mode 100644 include/linux/ahci-remap.h
>
> diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
> new file mode 100644
> index 0000000..883d366
> --- /dev/null
> +++ b/include/linux/ahci-remap.h
> @@ -0,0 +1,29 @@
> +#ifndef _LINUX_AHCI_REMAP_H
> +#define _LINUX_AHCI_REMAP_H
> +
> +#include <linux/sizes.h>
> +
> +#define AHCI_VSCAP		0xa4
> +#define AHCI_REMAP_CAP		0x800
> +#define AHCI_VSCAP		0xa4

    Why do it twice?

MBR, Sergei

Re: [PATCH 2/3] ahci-remap.h: add ahci remapping definitions

[Christoph Hellwig]

On Sat, Dec 03, 2016 at 01:57:01PM +0300, Sergei Shtylyov wrote:
>> +#ifndef _LINUX_AHCI_REMAP_H
>> +#define _LINUX_AHCI_REMAP_H
>> +
>> +#include <linux/sizes.h>
>> +
>> +#define AHCI_VSCAP		0xa4
>> +#define AHCI_REMAP_CAP		0x800
>> +#define AHCI_VSCAP		0xa4
>
>    Why do it twice?

No good reason, probably just me messing up the move..

[PATCH 2/3] ahci-remap.h: add ahci remapping definitions

[Christoph Hellwig]

On Sat, Dec 03, 2016@01:57:01PM +0300, Sergei Shtylyov wrote:
>> +#ifndef _LINUX_AHCI_REMAP_H
>> +#define _LINUX_AHCI_REMAP_H
>> +
>> +#include <linux/sizes.h>
>> +
>> +#define AHCI_VSCAP		0xa4
>> +#define AHCI_REMAP_CAP		0x800
>> +#define AHCI_VSCAP		0xa4
>
>    Why do it twice?

No good reason, probably just me messing up the move..

Re: detect and warn about ahci-remapped NVMe devices

[Tejun Heo]

Hello,

On Fri, Dec 02, 2016 at 07:31:00PM +0100, Christoph Hellwig wrote:
> Just a small band-aid to detect the AHCI-remapped NVMe devices
> for now.  Hopefully we'll eventually get a fake PCIe root port
> driver ala VMD for them.

Applied 1-3 to libata/for-4.10.  Fixed the dup constant definition in
0002.

Thanks.

-- 
tejun

detect and warn about ahci-remapped NVMe devices

[Tejun Heo]

Hello,

On Fri, Dec 02, 2016@07:31:00PM +0100, Christoph Hellwig wrote:
> Just a small band-aid to detect the AHCI-remapped NVMe devices
> for now.  Hopefully we'll eventually get a fake PCIe root port
> driver ala VMD for them.

Applied 1-3 to libata/for-4.10.  Fixed the dup constant definition in
0002.

Thanks.

-- 
tejun