From: Logan Gunthorpe <logang@deltatee.com>
To: Christoph Hellwig <hch@lst.de>, Sagi Grimberg <sagi@grimberg.me>,
"James E.J. Bottomley" <jejb@linux.vnet.ibm.com>,
"Martin K. Petersen" <martin.petersen@oracle.com>,
Jens Axboe <axboe@kernel.dk>,
Steve Wise <swise@opengridcomputing.com>,
Stephen Bates <sbates@raithlin.com>,
Max Gurtovoy <maxg@mellanox.com>,
Dan Williams <dan.j.williams@intel.com>,
Keith Busch <keith.busch@intel.com>,
Jason Gunthorpe <jgunthorpe@obsidianresearch.com>
Cc: linux-pci@vger.kernel.org, linux-scsi@vger.kernel.org,
linux-nvme@lists.infradead.org, linux-rdma@vger.kernel.org,
linux-nvdimm@ml01.01.org, linux-kernel@vger.kernel.org,
Logan Gunthorpe <logang@deltatee.com>
Subject: [RFC 2/8] cxgb4: setup pcie memory window 4 and create p2pmem region
Date: Thu, 30 Mar 2017 16:12:33 -0600 [thread overview]
Message-ID: <1490911959-5146-3-git-send-email-logang@deltatee.com> (raw)
In-Reply-To: <1490911959-5146-1-git-send-email-logang@deltatee.com>
From: Steve Wise <swise@opengridcomputing.com>
Some cxgb4 cards expose memory as part of BAR4. This patch registers
this memory as a p2pmem device.
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
Signed-off-by: Stephen Bates <sbates@raithlin.com>
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 3 +
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 97 ++++++++++++++++++++++++-
drivers/net/ethernet/chelsio/cxgb4/t4_regs.h | 5 ++
3 files changed, 102 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 163543b..e92443b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -48,6 +48,7 @@
#include <linux/vmalloc.h>
#include <linux/etherdevice.h>
#include <linux/net_tstamp.h>
+#include <linux/p2pmem.h>
#include <asm/io.h>
#include "t4_chip_type.h"
#include "cxgb4_uld.h"
@@ -859,6 +860,8 @@ struct adapter {
/* TC u32 offload */
struct cxgb4_tc_u32_table *tc_u32;
+
+ struct p2pmem_dev *p2pmem;
};
/* Support for "sched-class" command to allow a TX Scheduling Class to be
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index afb0967..a33bcd1 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -172,6 +172,11 @@ module_param(select_queue, int, 0644);
MODULE_PARM_DESC(select_queue,
"Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
+static bool use_p2pmem;
+module_param(use_p2pmem, bool, 0644);
+MODULE_PARM_DESC(use_p2pmem,
+ "Enable registering a p2pmem device with bar space (if available)");
+
static struct dentry *cxgb4_debugfs_root;
LIST_HEAD(adapter_list);
@@ -2835,6 +2840,54 @@ static void setup_memwin_rdma(struct adapter *adap)
}
}
+static void setup_memwin_p2pmem(struct adapter *adap)
+{
+ unsigned int mem_base = t4_read_reg(adap, CIM_EXTMEM2_BASE_ADDR_A);
+ unsigned int mem_size = t4_read_reg(adap, CIM_EXTMEM2_ADDR_SIZE_A);
+
+ if (!use_p2pmem)
+ return;
+
+ if (mem_base != 0 && mem_size != 0) {
+ unsigned int sz_kb, pcieofst;
+
+ sz_kb = roundup_pow_of_two(mem_size) >> 10;
+
+ /*
+ * The start offset must be aligned to the window size.
+ * Also, BAR4 has MSIX vectors using the first 8KB.
+ * Further, the min allowed p2pmem region size is 1MB,
+ * so set the start offset to the memory size and we're aligned
+ * as well as past the 8KB vector table.
+ */
+ pcieofst = sz_kb << 10;
+
+ dev_info(adap->pdev_dev,
+ "p2pmem base 0x%x, size %uB, ilog2(sk_kb) 0x%x, "
+ "pcieofst 0x%X\n", mem_base, mem_size, ilog2(sz_kb),
+ pcieofst);
+
+ /* Write the window offset and size */
+ t4_write_reg(adap,
+ PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A,
+ MEMWIN_RSVD4),
+ pcieofst | BIR_V(2) | WINDOW_V(ilog2(sz_kb)));
+
+ /* Write the adapter memory base/start */
+ t4_write_reg(adap,
+ PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A,
+ MEMWIN_RSVD4),
+ MEMOFST_V((mem_base >> MEMOFST_S)) | PFNUM_V(adap->pf));
+
+ /* Read it back to flush it */
+ t4_read_reg(adap,
+ PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A,
+ MEMWIN_RSVD4));
+ } else
+ dev_info(adap->pdev_dev, "p2pmem memory not reserved, "
+ "base 0x%x size %uB\n", mem_base, mem_size);
+}
+
static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
{
u32 v;
@@ -4622,6 +4675,42 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
}
#endif
+static int init_p2pmem(struct adapter *adapter)
+{
+ unsigned int mem_size = t4_read_reg(adapter, CIM_EXTMEM2_ADDR_SIZE_A);
+ struct p2pmem_dev *p;
+ int rc;
+ struct resource res;
+
+ if (!mem_size || !use_p2pmem)
+ return 0;
+
+ mem_size = roundup_pow_of_two(mem_size);
+
+ /*
+ * Create a subset of BAR4 for the p2pmem region based on the
+ * exported memory size.
+ */
+ memcpy(&res, &adapter->pdev->resource[4], sizeof(res));
+ res.start += mem_size;
+ res.end = res.start + mem_size - 1;
+ dev_info(adapter->pdev_dev, "p2pmem resource start 0x%llx end 0x%llx size %lluB\n",
+ res.start, res.end, resource_size(&res));
+
+ p = p2pmem_create(&adapter->pdev->dev);
+ if (IS_ERR(p))
+ return PTR_ERR(p);
+
+ rc = p2pmem_add_resource(p, &res);
+ if (rc) {
+ p2pmem_unregister(p);
+ return rc;
+ }
+ adapter->p2pmem = p;
+
+ return 0;
+}
+
static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
int func, i, err, s_qpp, qpp, num_seg;
@@ -4784,8 +4873,8 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz);
#endif
setup_memwin_rdma(adapter);
- if (err)
- goto out_unmap_bar;
+
+ setup_memwin_p2pmem(adapter);
/* configure SGE_STAT_CFG_A to read WC stats */
if (!is_t4(adapter->params.chip))
@@ -4989,6 +5078,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
print_adapter_info(adapter);
setup_fw_sge_queues(adapter);
+ init_p2pmem(adapter);
return 0;
sriov:
@@ -5047,7 +5137,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
free_msix_info(adapter);
if (adapter->num_uld || adapter->num_ofld_uld)
t4_uld_mem_free(adapter);
- out_unmap_bar:
if (!is_t4(adapter->params.chip))
iounmap(adapter->bar2);
out_free_adapter:
@@ -5075,6 +5164,8 @@ static void remove_one(struct pci_dev *pdev)
return;
}
+ p2pmem_unregister(adapter->p2pmem);
+
if (adapter->pf == 4) {
int i;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
index 3348d33..199ddfb 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
@@ -735,6 +735,11 @@
#define PCIE_MEM_ACCESS_OFFSET_A 0x306c
+#define MEMOFST_S 7
+#define MEMOFST_M 0x1ffffffU
+#define MEMOFST_V(x) ((x) << MEMOFST_S)
+#define MEMOFST_G(x) (((x) >> MEMOFST_S) & MEMOFST_M)
+
#define ENABLE_S 30
#define ENABLE_V(x) ((x) << ENABLE_S)
#define ENABLE_F ENABLE_V(1U)
--
2.1.4
next prev parent reply other threads:[~2017-03-30 22:13 UTC|newest]
Thread overview: 81+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-03-30 22:12 [RFC 0/8] Copy Offload with Peer-to-Peer PCI Memory Logan Gunthorpe
2017-03-30 22:12 ` [RFC 1/8] Introduce Peer-to-Peer memory (p2pmem) device Logan Gunthorpe
2017-03-31 18:49 ` Sinan Kaya
2017-03-31 21:23 ` Logan Gunthorpe
2017-03-31 21:38 ` Sinan Kaya
2017-03-31 22:42 ` Logan Gunthorpe
2017-03-31 23:51 ` Sinan Kaya
2017-04-01 1:57 ` Logan Gunthorpe
2017-04-01 2:17 ` okaya
2017-04-01 22:16 ` Logan Gunthorpe
2017-04-02 2:26 ` Sinan Kaya
2017-04-02 17:21 ` Logan Gunthorpe
2017-04-02 21:03 ` Sinan Kaya
2017-04-03 4:26 ` Logan Gunthorpe
2017-04-25 11:58 ` Marta Rybczynska
2017-04-25 16:58 ` Logan Gunthorpe
2017-03-30 22:12 ` Logan Gunthorpe [this message]
2017-04-04 10:42 ` [RFC 2/8] cxgb4: setup pcie memory window 4 and create p2pmem region Sagi Grimberg
2017-04-04 15:56 ` Logan Gunthorpe
2017-04-05 15:41 ` Steve Wise
2017-03-30 22:12 ` [RFC 3/8] nvmet: Use p2pmem in nvme target Logan Gunthorpe
2017-04-04 10:40 ` Sagi Grimberg
2017-04-04 16:16 ` Logan Gunthorpe
2017-04-06 5:47 ` Sagi Grimberg
2017-04-06 15:52 ` Logan Gunthorpe
2017-03-30 22:12 ` [RFC 4/8] p2pmem: Add debugfs "stats" file Logan Gunthorpe
2017-04-04 10:46 ` Sagi Grimberg
2017-04-04 17:25 ` Logan Gunthorpe
2017-04-05 15:43 ` Steve Wise
2017-03-30 22:12 ` [RFC 5/8] scatterlist: Modify SG copy functions to support io memory Logan Gunthorpe
2017-03-31 7:09 ` Christoph Hellwig
2017-03-31 15:41 ` Logan Gunthorpe
2017-04-03 21:20 ` Logan Gunthorpe
2017-04-03 21:44 ` Dan Williams
2017-04-03 22:10 ` Logan Gunthorpe
2017-04-03 22:47 ` Dan Williams
2017-04-03 23:12 ` Logan Gunthorpe
2017-04-04 0:07 ` Dan Williams
2017-04-07 17:59 ` Logan Gunthorpe
2017-03-30 22:12 ` [RFC 6/8] nvmet: Be careful about using iomem accesses when dealing with p2pmem Logan Gunthorpe
2017-04-04 10:59 ` Sagi Grimberg
2017-04-04 15:46 ` Jason Gunthorpe
2017-04-04 17:21 ` Logan Gunthorpe
2017-04-06 5:33 ` Sagi Grimberg
2017-04-06 16:02 ` Logan Gunthorpe
2017-04-06 16:35 ` Jason Gunthorpe
2017-04-07 11:19 ` Stephen Bates
2017-04-10 8:29 ` Sagi Grimberg
2017-04-10 16:03 ` Logan Gunthorpe
2017-03-30 22:12 ` [RFC 7/8] p2pmem: Support device removal Logan Gunthorpe
2017-03-30 22:12 ` [RFC 8/8] p2pmem: Added char device user interface Logan Gunthorpe
2017-04-12 5:22 ` [RFC 0/8] Copy Offload with Peer-to-Peer PCI Memory Benjamin Herrenschmidt
2017-04-12 17:09 ` Logan Gunthorpe
2017-04-12 21:55 ` Benjamin Herrenschmidt
2017-04-13 21:22 ` Logan Gunthorpe
2017-04-13 22:37 ` Benjamin Herrenschmidt
2017-04-13 23:26 ` Bjorn Helgaas
2017-04-14 4:16 ` Jason Gunthorpe
2017-04-14 4:40 ` Logan Gunthorpe
2017-04-14 11:37 ` Benjamin Herrenschmidt
2017-04-14 11:39 ` Benjamin Herrenschmidt
2017-04-14 11:37 ` Benjamin Herrenschmidt
2017-04-14 17:30 ` Logan Gunthorpe
2017-04-14 19:04 ` Bjorn Helgaas
2017-04-14 22:07 ` Benjamin Herrenschmidt
2017-04-15 17:41 ` Logan Gunthorpe
2017-04-15 22:09 ` Dan Williams
2017-04-16 3:01 ` Benjamin Herrenschmidt
2017-04-16 4:46 ` Logan Gunthorpe
2017-04-16 15:53 ` Dan Williams
2017-04-16 16:34 ` Logan Gunthorpe
2017-04-16 22:31 ` Benjamin Herrenschmidt
2017-04-24 7:36 ` Knut Omang
2017-04-24 16:14 ` Logan Gunthorpe
2017-04-25 6:30 ` Knut Omang
2017-04-25 17:03 ` Logan Gunthorpe
2017-04-25 21:23 ` Stephen Bates
2017-04-25 21:23 ` Stephen Bates
2017-04-16 22:26 ` Benjamin Herrenschmidt
2017-04-15 22:17 ` Benjamin Herrenschmidt
2017-04-16 5:36 ` Logan Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1490911959-5146-3-git-send-email-logang@deltatee.com \
--to=logang@deltatee.com \
--cc=axboe@kernel.dk \
--cc=dan.j.williams@intel.com \
--cc=hch@lst.de \
--cc=jejb@linux.vnet.ibm.com \
--cc=jgunthorpe@obsidianresearch.com \
--cc=keith.busch@intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nvdimm@ml01.01.org \
--cc=linux-nvme@lists.infradead.org \
--cc=linux-pci@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=linux-scsi@vger.kernel.org \
--cc=martin.petersen@oracle.com \
--cc=maxg@mellanox.com \
--cc=sagi@grimberg.me \
--cc=sbates@raithlin.com \
--cc=swise@opengridcomputing.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).