From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ale.deltatee.com (ale.deltatee.com [207.54.116.67]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id 0F22E22512138 for ; Mon, 23 Apr 2018 16:31:00 -0700 (PDT) From: Logan Gunthorpe Date: Mon, 23 Apr 2018 17:30:43 -0600 Message-Id: <20180423233046.21476-12-logang@deltatee.com> In-Reply-To: <20180423233046.21476-1-logang@deltatee.com> References: <20180423233046.21476-1-logang@deltatee.com> Subject: [PATCH v4 11/14] nvme-pci: Add a quirk for a pseudo CMB List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: linux-nvdimm-bounces@lists.01.org Sender: "Linux-nvdimm" To: linux-kernel@vger.kernel.org, linux-pci@vger.kernel.org, linux-nvme@lists.infradead.org, linux-rdma@vger.kernel.org, linux-nvdimm@lists.01.org, linux-block@vger.kernel.org Cc: Jens Axboe , =?UTF-8?q?Christian=20K=C3=B6nig?= , Benjamin Herrenschmidt , Alex Williamson , Keith Busch , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= , Jason Gunthorpe , Bjorn Helgaas , Max Gurtovoy , Christoph Hellwig List-ID: Introduce a quirk to use CMB-like memory on older devices that have an exposed BAR but do not advertise support for using CMBLOC and CMBSIZE. We'd like to use some of these older cards to test P2P memory. Signed-off-by: Logan Gunthorpe Reviewed-by: Sagi Grimberg --- drivers/nvme/host/nvme.h | 7 +++++++ drivers/nvme/host/pci.c | 24 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 9a689c13998f..885e9ec9b889 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -84,6 +84,13 @@ enum nvme_quirks { * Supports the LighNVM command set if indicated in vs[1]. */ NVME_QUIRK_LIGHTNVM = (1 << 6), + + /* + * Pseudo CMB Support on BAR 4. For adapters like the Microsemi + * NVRAM that have CMB-like memory on a BAR but does not set + * CMBLOC or CMBSZ. + */ + NVME_QUIRK_PSEUDO_CMB_BAR4 = (1 << 7), }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 09b6aba6ed28..e526e969680a 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1685,6 +1685,13 @@ static ssize_t nvme_cmb_show(struct device *dev, } static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL); +static u32 nvme_pseudo_cmbsz(struct pci_dev *pdev, int bar) +{ + return NVME_CMBSZ_WDS | NVME_CMBSZ_RDS | + (((ilog2(SZ_16M) - 12) / 4) << NVME_CMBSZ_SZU_SHIFT) | + ((pci_resource_len(pdev, bar) / SZ_16M) << NVME_CMBSZ_SZ_SHIFT); +} + static u64 nvme_cmb_size_unit(struct nvme_dev *dev) { u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK; @@ -1704,10 +1711,15 @@ static void nvme_map_cmb(struct nvme_dev *dev) struct pci_dev *pdev = to_pci_dev(dev->dev); int bar; - dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); - if (!dev->cmbsz) - return; - dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + if (dev->ctrl.quirks & NVME_QUIRK_PSEUDO_CMB_BAR4) { + dev->cmbsz = nvme_pseudo_cmbsz(pdev, 4); + dev->cmbloc = 4; + } else { + dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); + if (!dev->cmbsz) + return; + dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + } size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev); offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc); @@ -2736,6 +2748,10 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_LIGHTNVM, }, { PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */ .driver_data = NVME_QUIRK_LIGHTNVM, }, + { PCI_DEVICE(0x11f8, 0xf117), /* Microsemi NVRAM adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, + { PCI_DEVICE(0x1db1, 0x0002), /* Everspin nvNitro adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) }, -- 2.11.0 _______________________________________________ Linux-nvdimm mailing list Linux-nvdimm@lists.01.org https://lists.01.org/mailman/listinfo/linux-nvdimm From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ale.deltatee.com ([207.54.116.67]:41920 "EHLO ale.deltatee.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S932652AbeDWXbF (ORCPT ); Mon, 23 Apr 2018 19:31:05 -0400 From: Logan Gunthorpe To: linux-kernel@vger.kernel.org, linux-pci@vger.kernel.org, linux-nvme@lists.infradead.org, linux-rdma@vger.kernel.org, linux-nvdimm@lists.01.org, linux-block@vger.kernel.org Cc: Stephen Bates , Christoph Hellwig , Jens Axboe , Keith Busch , Sagi Grimberg , Bjorn Helgaas , Jason Gunthorpe , Max Gurtovoy , Dan Williams , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= , Benjamin Herrenschmidt , Alex Williamson , =?UTF-8?q?Christian=20K=C3=B6nig?= , Logan Gunthorpe Date: Mon, 23 Apr 2018 17:30:43 -0600 Message-Id: <20180423233046.21476-12-logang@deltatee.com> In-Reply-To: <20180423233046.21476-1-logang@deltatee.com> References: <20180423233046.21476-1-logang@deltatee.com> Subject: [PATCH v4 11/14] nvme-pci: Add a quirk for a pseudo CMB Sender: linux-block-owner@vger.kernel.org List-Id: linux-block@vger.kernel.org Introduce a quirk to use CMB-like memory on older devices that have an exposed BAR but do not advertise support for using CMBLOC and CMBSIZE. We'd like to use some of these older cards to test P2P memory. Signed-off-by: Logan Gunthorpe Reviewed-by: Sagi Grimberg --- drivers/nvme/host/nvme.h | 7 +++++++ drivers/nvme/host/pci.c | 24 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 9a689c13998f..885e9ec9b889 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -84,6 +84,13 @@ enum nvme_quirks { * Supports the LighNVM command set if indicated in vs[1]. */ NVME_QUIRK_LIGHTNVM = (1 << 6), + + /* + * Pseudo CMB Support on BAR 4. For adapters like the Microsemi + * NVRAM that have CMB-like memory on a BAR but does not set + * CMBLOC or CMBSZ. + */ + NVME_QUIRK_PSEUDO_CMB_BAR4 = (1 << 7), }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 09b6aba6ed28..e526e969680a 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1685,6 +1685,13 @@ static ssize_t nvme_cmb_show(struct device *dev, } static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL); +static u32 nvme_pseudo_cmbsz(struct pci_dev *pdev, int bar) +{ + return NVME_CMBSZ_WDS | NVME_CMBSZ_RDS | + (((ilog2(SZ_16M) - 12) / 4) << NVME_CMBSZ_SZU_SHIFT) | + ((pci_resource_len(pdev, bar) / SZ_16M) << NVME_CMBSZ_SZ_SHIFT); +} + static u64 nvme_cmb_size_unit(struct nvme_dev *dev) { u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK; @@ -1704,10 +1711,15 @@ static void nvme_map_cmb(struct nvme_dev *dev) struct pci_dev *pdev = to_pci_dev(dev->dev); int bar; - dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); - if (!dev->cmbsz) - return; - dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + if (dev->ctrl.quirks & NVME_QUIRK_PSEUDO_CMB_BAR4) { + dev->cmbsz = nvme_pseudo_cmbsz(pdev, 4); + dev->cmbloc = 4; + } else { + dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); + if (!dev->cmbsz) + return; + dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + } size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev); offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc); @@ -2736,6 +2748,10 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_LIGHTNVM, }, { PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */ .driver_data = NVME_QUIRK_LIGHTNVM, }, + { PCI_DEVICE(0x11f8, 0xf117), /* Microsemi NVRAM adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, + { PCI_DEVICE(0x1db1, 0x0002), /* Everspin nvNitro adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) }, -- 2.11.0 From mboxrd@z Thu Jan 1 00:00:00 1970 From: Logan Gunthorpe Subject: [PATCH v4 11/14] nvme-pci: Add a quirk for a pseudo CMB Date: Mon, 23 Apr 2018 17:30:43 -0600 Message-ID: <20180423233046.21476-12-logang@deltatee.com> References: <20180423233046.21476-1-logang@deltatee.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <20180423233046.21476-1-logang-OTvnGxWRz7hWk0Htik3J/w@public.gmane.org> List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: linux-nvdimm-bounces-hn68Rpc1hR1g9hUCZPvPmw@public.gmane.org Sender: "Linux-nvdimm" To: linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, linux-pci-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, linux-nvme-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, linux-nvdimm-hn68Rpc1hR1g9hUCZPvPmw@public.gmane.org, linux-block-u79uwXL29TY76Z2rM5mHXA@public.gmane.org Cc: Jens Axboe , =?UTF-8?q?Christian=20K=C3=B6nig?= , Benjamin Herrenschmidt , Alex Williamson , Keith Busch , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= , Jason Gunthorpe , Bjorn Helgaas , Max Gurtovoy , Christoph Hellwig List-Id: linux-rdma@vger.kernel.org Introduce a quirk to use CMB-like memory on older devices that have an exposed BAR but do not advertise support for using CMBLOC and CMBSIZE. We'd like to use some of these older cards to test P2P memory. Signed-off-by: Logan Gunthorpe Reviewed-by: Sagi Grimberg --- drivers/nvme/host/nvme.h | 7 +++++++ drivers/nvme/host/pci.c | 24 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 9a689c13998f..885e9ec9b889 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -84,6 +84,13 @@ enum nvme_quirks { * Supports the LighNVM command set if indicated in vs[1]. */ NVME_QUIRK_LIGHTNVM = (1 << 6), + + /* + * Pseudo CMB Support on BAR 4. For adapters like the Microsemi + * NVRAM that have CMB-like memory on a BAR but does not set + * CMBLOC or CMBSZ. + */ + NVME_QUIRK_PSEUDO_CMB_BAR4 = (1 << 7), }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 09b6aba6ed28..e526e969680a 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1685,6 +1685,13 @@ static ssize_t nvme_cmb_show(struct device *dev, } static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL); +static u32 nvme_pseudo_cmbsz(struct pci_dev *pdev, int bar) +{ + return NVME_CMBSZ_WDS | NVME_CMBSZ_RDS | + (((ilog2(SZ_16M) - 12) / 4) << NVME_CMBSZ_SZU_SHIFT) | + ((pci_resource_len(pdev, bar) / SZ_16M) << NVME_CMBSZ_SZ_SHIFT); +} + static u64 nvme_cmb_size_unit(struct nvme_dev *dev) { u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK; @@ -1704,10 +1711,15 @@ static void nvme_map_cmb(struct nvme_dev *dev) struct pci_dev *pdev = to_pci_dev(dev->dev); int bar; - dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); - if (!dev->cmbsz) - return; - dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + if (dev->ctrl.quirks & NVME_QUIRK_PSEUDO_CMB_BAR4) { + dev->cmbsz = nvme_pseudo_cmbsz(pdev, 4); + dev->cmbloc = 4; + } else { + dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); + if (!dev->cmbsz) + return; + dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + } size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev); offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc); @@ -2736,6 +2748,10 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_LIGHTNVM, }, { PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */ .driver_data = NVME_QUIRK_LIGHTNVM, }, + { PCI_DEVICE(0x11f8, 0xf117), /* Microsemi NVRAM adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, + { PCI_DEVICE(0x1db1, 0x0002), /* Everspin nvNitro adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) }, -- 2.11.0 From mboxrd@z Thu Jan 1 00:00:00 1970 From: logang@deltatee.com (Logan Gunthorpe) Date: Mon, 23 Apr 2018 17:30:43 -0600 Subject: [PATCH v4 11/14] nvme-pci: Add a quirk for a pseudo CMB In-Reply-To: <20180423233046.21476-1-logang@deltatee.com> References: <20180423233046.21476-1-logang@deltatee.com> Message-ID: <20180423233046.21476-12-logang@deltatee.com> Introduce a quirk to use CMB-like memory on older devices that have an exposed BAR but do not advertise support for using CMBLOC and CMBSIZE. We'd like to use some of these older cards to test P2P memory. Signed-off-by: Logan Gunthorpe Reviewed-by: Sagi Grimberg --- drivers/nvme/host/nvme.h | 7 +++++++ drivers/nvme/host/pci.c | 24 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 9a689c13998f..885e9ec9b889 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -84,6 +84,13 @@ enum nvme_quirks { * Supports the LighNVM command set if indicated in vs[1]. */ NVME_QUIRK_LIGHTNVM = (1 << 6), + + /* + * Pseudo CMB Support on BAR 4. For adapters like the Microsemi + * NVRAM that have CMB-like memory on a BAR but does not set + * CMBLOC or CMBSZ. + */ + NVME_QUIRK_PSEUDO_CMB_BAR4 = (1 << 7), }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 09b6aba6ed28..e526e969680a 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1685,6 +1685,13 @@ static ssize_t nvme_cmb_show(struct device *dev, } static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL); +static u32 nvme_pseudo_cmbsz(struct pci_dev *pdev, int bar) +{ + return NVME_CMBSZ_WDS | NVME_CMBSZ_RDS | + (((ilog2(SZ_16M) - 12) / 4) << NVME_CMBSZ_SZU_SHIFT) | + ((pci_resource_len(pdev, bar) / SZ_16M) << NVME_CMBSZ_SZ_SHIFT); +} + static u64 nvme_cmb_size_unit(struct nvme_dev *dev) { u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK; @@ -1704,10 +1711,15 @@ static void nvme_map_cmb(struct nvme_dev *dev) struct pci_dev *pdev = to_pci_dev(dev->dev); int bar; - dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); - if (!dev->cmbsz) - return; - dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + if (dev->ctrl.quirks & NVME_QUIRK_PSEUDO_CMB_BAR4) { + dev->cmbsz = nvme_pseudo_cmbsz(pdev, 4); + dev->cmbloc = 4; + } else { + dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); + if (!dev->cmbsz) + return; + dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); + } size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev); offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc); @@ -2736,6 +2748,10 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_LIGHTNVM, }, { PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */ .driver_data = NVME_QUIRK_LIGHTNVM, }, + { PCI_DEVICE(0x11f8, 0xf117), /* Microsemi NVRAM adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, + { PCI_DEVICE(0x1db1, 0x0002), /* Everspin nvNitro adaptor */ + .driver_data = NVME_QUIRK_PSEUDO_CMB_BAR4, }, { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) }, -- 2.11.0