All of lore.kernel.org
 help / color / mirror / Atom feed
From: Chenbo Xia <chenbo.xia@intel.com>
To: dev@dpdk.org, thomas@monjalon.net, cunming.liang@intel.com,
	jingjing.wu@intel.com
Cc: anatoly.burakov@intel.com, ferruh.yigit@intel.com, mdr@ashroe.eu,
	nhorman@tuxdriver.com, bruce.richardson@intel.com,
	david.marchand@redhat.com, stephen@networkplumber.org,
	konstantin.ananyev@intel.com, Tiwei Bie <tiwei.bie@intel.com>
Subject: [dpdk-dev] [RFC v3 3/6] bus/pci: introduce helper for MMIO read and write
Date: Tue,  1 Jun 2021 11:06:41 +0800	[thread overview]
Message-ID: <20210601030644.3318-4-chenbo.xia@intel.com> (raw)
In-Reply-To: <20210601030644.3318-1-chenbo.xia@intel.com>

From: Tiwei Bie <tiwei.bie@intel.com>

The MMIO regions may not be mmap-able for mediated PCI device.
In this case, the application should explicitly do read and write
to access these regions.

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 drivers/bus/pci/bsd/pci.c        | 22 +++++++++++++++
 drivers/bus/pci/linux/pci.c      | 46 ++++++++++++++++++++++++++++++
 drivers/bus/pci/linux/pci_init.h | 10 +++++++
 drivers/bus/pci/linux/pci_uio.c  | 22 +++++++++++++++
 drivers/bus/pci/linux/pci_vfio.c | 36 ++++++++++++++++++++++++
 drivers/bus/pci/rte_bus_pci.h    | 48 ++++++++++++++++++++++++++++++++
 drivers/bus/pci/version.map      |  4 +++
 7 files changed, 188 insertions(+)

diff --git a/drivers/bus/pci/bsd/pci.c b/drivers/bus/pci/bsd/pci.c
index 20ce979f60..781f65c637 100644
--- a/drivers/bus/pci/bsd/pci.c
+++ b/drivers/bus/pci/bsd/pci.c
@@ -494,6 +494,28 @@ int rte_pci_write_config(const struct rte_pci_device *dev,
 	return -1;
 }
 
+/* Read PCI MMIO space. */
+int rte_pci_mmio_read(const struct rte_pci_device *dev, int bar,
+		      void *buf, size_t len, off_t offset)
+{
+	if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL ||
+			(uint64_t)offset + len > dev->mem_resource[bar].len)
+		return -1;
+	memcpy(buf, (uint8_t *)dev->mem_resource[bar].addr + offset, len);
+	return len;
+}
+
+/* Write PCI MMIO space. */
+int rte_pci_mmio_write(const struct rte_pci_device *dev, int bar,
+		       const void *buf, size_t len, off_t offset)
+{
+	if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL ||
+			(uint64_t)offset + len > dev->mem_resource[bar].len)
+		return -1;
+	memcpy((uint8_t *)dev->mem_resource[bar].addr + offset, buf, len);
+	return len;
+}
+
 int
 rte_pci_ioport_map(struct rte_pci_device *dev, int bar,
 		struct rte_pci_ioport *p)
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index 8f1fddbf20..4805f277c5 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -682,6 +682,52 @@ int rte_pci_write_config(const struct rte_pci_device *device,
 	}
 }
 
+/* Read PCI MMIO space. */
+int rte_pci_mmio_read(const struct rte_pci_device *device, int bar,
+		void *buf, size_t len, off_t offset)
+{
+	char devname[RTE_DEV_NAME_MAX_LEN] = "";
+
+	switch (device->kdrv) {
+	case RTE_PCI_KDRV_IGB_UIO:
+	case RTE_PCI_KDRV_UIO_GENERIC:
+		return pci_uio_mmio_read(device, bar, buf, len, offset);
+#ifdef VFIO_PRESENT
+	case RTE_PCI_KDRV_VFIO:
+		return pci_vfio_mmio_read(device, bar, buf, len, offset);
+#endif
+	default:
+		rte_pci_device_name(&device->addr, devname,
+				    RTE_DEV_NAME_MAX_LEN);
+		RTE_LOG(ERR, EAL,
+			"Unknown driver type for %s\n", devname);
+		return -1;
+	}
+}
+
+/* Write PCI MMIO space. */
+int rte_pci_mmio_write(const struct rte_pci_device *device, int bar,
+		const void *buf, size_t len, off_t offset)
+{
+	char devname[RTE_DEV_NAME_MAX_LEN] = "";
+
+	switch (device->kdrv) {
+	case RTE_PCI_KDRV_IGB_UIO:
+	case RTE_PCI_KDRV_UIO_GENERIC:
+		return pci_uio_mmio_write(device, bar, buf, len, offset);
+#ifdef VFIO_PRESENT
+	case RTE_PCI_KDRV_VFIO:
+		return pci_vfio_mmio_write(device, bar, buf, len, offset);
+#endif
+	default:
+		rte_pci_device_name(&device->addr, devname,
+				    RTE_DEV_NAME_MAX_LEN);
+		RTE_LOG(ERR, EAL,
+			"Unknown driver type for %s\n", devname);
+		return -1;
+	}
+}
+
 int
 rte_pci_ioport_map(struct rte_pci_device *dev, int bar,
 		struct rte_pci_ioport *p)
diff --git a/drivers/bus/pci/linux/pci_init.h b/drivers/bus/pci/linux/pci_init.h
index 9f6659ba6e..6853fa88a3 100644
--- a/drivers/bus/pci/linux/pci_init.h
+++ b/drivers/bus/pci/linux/pci_init.h
@@ -37,6 +37,11 @@ int pci_uio_read_config(const struct rte_intr_handle *intr_handle,
 int pci_uio_write_config(const struct rte_intr_handle *intr_handle,
 			 const void *buf, size_t len, off_t offs);
 
+int pci_uio_mmio_read(const struct rte_pci_device *dev, int bar,
+		      void *buf, size_t len, off_t offset);
+int pci_uio_mmio_write(const struct rte_pci_device *dev, int bar,
+		       const void *buf, size_t len, off_t offset);
+
 int pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
 		       struct rte_pci_ioport *p);
 void pci_uio_ioport_read(struct rte_pci_ioport *p,
@@ -71,6 +76,11 @@ int pci_vfio_read_config(const struct rte_pci_device *dev,
 int pci_vfio_write_config(const struct rte_pci_device *dev,
 			  const void *buf, size_t len, off_t offs);
 
+int pci_vfio_mmio_read(const struct rte_pci_device *dev, int bar,
+		       void *buf, size_t len, off_t offset);
+int pci_vfio_mmio_write(const struct rte_pci_device *dev, int bar,
+			const void *buf, size_t len, off_t offset);
+
 int pci_vfio_ioport_map(struct rte_pci_device *dev, int bar,
 		        struct rte_pci_ioport *p);
 void pci_vfio_ioport_read(struct rte_pci_ioport *p,
diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c
index 39ebeac2a0..2482635058 100644
--- a/drivers/bus/pci/linux/pci_uio.c
+++ b/drivers/bus/pci/linux/pci_uio.c
@@ -45,6 +45,28 @@ pci_uio_write_config(const struct rte_intr_handle *intr_handle,
 	return pwrite(intr_handle->uio_cfg_fd, buf, len, offset);
 }
 
+int
+pci_uio_mmio_read(const struct rte_pci_device *dev, int bar,
+		  void *buf, size_t len, off_t offset)
+{
+	if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL ||
+			(uint64_t)offset + len > dev->mem_resource[bar].len)
+		return -1;
+	memcpy(buf, (uint8_t *)dev->mem_resource[bar].addr + offset, len);
+	return len;
+}
+
+int
+pci_uio_mmio_write(const struct rte_pci_device *dev, int bar,
+		   const void *buf, size_t len, off_t offset)
+{
+	if (bar >= PCI_MAX_RESOURCE || dev->mem_resource[bar].addr == NULL ||
+			(uint64_t)offset + len > dev->mem_resource[bar].len)
+		return -1;
+	memcpy((uint8_t *)dev->mem_resource[bar].addr + offset, buf, len);
+	return len;
+}
+
 static int
 pci_uio_set_bus_master(int dev_fd)
 {
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 012e7f72c1..3ecd984215 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -1212,6 +1212,42 @@ pci_vfio_ioport_unmap(struct rte_pci_ioport *p)
 	return -1;
 }
 
+int
+pci_vfio_mmio_read(const struct rte_pci_device *dev, int bar,
+		   void *buf, size_t len, off_t offs)
+{
+	uint64_t size, offset;
+	int fd;
+
+	fd = dev->intr_handle.vfio_dev_fd;
+
+	if (pci_vfio_get_region(dev, bar, &size, &offset) != 0)
+		return -1;
+
+	if ((uint64_t)len + offs > size)
+		return -1;
+
+	return pread64(fd, buf, len, offset + offs);
+}
+
+int
+pci_vfio_mmio_write(const struct rte_pci_device *dev, int bar,
+		    const void *buf, size_t len, off_t offs)
+{
+	uint64_t size, offset;
+	int fd;
+
+	fd = dev->intr_handle.vfio_dev_fd;
+
+	if (pci_vfio_get_region(dev, bar, &size, &offset) != 0)
+		return -1;
+
+	if ((uint64_t)len + offs > size)
+		return -1;
+
+	return pwrite64(fd, buf, len, offset + offs);
+}
+
 int
 pci_vfio_is_enabled(void)
 {
diff --git a/drivers/bus/pci/rte_bus_pci.h b/drivers/bus/pci/rte_bus_pci.h
index 64886b4731..dc26811b0a 100644
--- a/drivers/bus/pci/rte_bus_pci.h
+++ b/drivers/bus/pci/rte_bus_pci.h
@@ -310,6 +310,54 @@ int rte_pci_read_config(const struct rte_pci_device *device,
 int rte_pci_write_config(const struct rte_pci_device *device,
 		const void *buf, size_t len, off_t offset);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Read from a MMIO pci resource.
+ *
+ * @param device
+ *   A pointer to a rte_pci_device structure describing the device
+ *   to use
+ * @param bar
+ *   Index of the io pci resource we want to access.
+ * @param buf
+ *   A data buffer where the bytes should be read into
+ * @param len
+ *   The length of the data buffer.
+ * @param offset
+ *   The offset into MMIO space described by @bar
+ * @return
+ *  Number of bytes read on success, negative on error.
+ */
+__rte_experimental
+int rte_pci_mmio_read(const struct rte_pci_device *device, int bar,
+		void *buf, size_t len, off_t offset);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Write to a MMIO pci resource.
+ *
+ * @param device
+ *   A pointer to a rte_pci_device structure describing the device
+ *   to use
+ * @param bar
+ *   Index of the io pci resource we want to access.
+ * @param buf
+ *   A data buffer containing the bytes should be written
+ * @param len
+ *   The length of the data buffer.
+ * @param offset
+ *   The offset into MMIO space described by @bar
+ * @return
+ *  Number of bytes written on success, negative on error.
+ */
+__rte_experimental
+int rte_pci_mmio_write(const struct rte_pci_device *device, int bar,
+		const void *buf, size_t len, off_t offset);
+
 /**
  * A structure used to access io resources for a pci device.
  * rte_pci_ioport is arch, os, driver specific, and should not be used outside
diff --git a/drivers/bus/pci/version.map b/drivers/bus/pci/version.map
index f33ed0abd1..02e4219aab 100644
--- a/drivers/bus/pci/version.map
+++ b/drivers/bus/pci/version.map
@@ -21,4 +21,8 @@ EXPERIMENTAL {
 	global:
 
 	rte_pci_find_ext_capability;
+
+	# added in 21.08
+	rte_pci_mmio_read;
+	rte_pci_mmio_write;
 };
-- 
2.17.1


  parent reply	other threads:[~2021-06-01  3:17 UTC|newest]

Thread overview: 33+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-04-03  7:18 [RFC 0/3] Add mdev (Mediated device) support in DPDK Tiwei Bie
2019-04-03  7:18 ` [RFC 1/3] eal: add a helper for reading string from sysfs Tiwei Bie
2019-04-03  7:18 ` [RFC 2/3] bus/mdev: add mdev bus support Tiwei Bie
2019-04-03  7:18 ` [RFC 3/3] bus/pci: add mdev support Tiwei Bie
2019-04-03 14:13   ` Wiles, Keith
2019-04-04  4:19     ` Tiwei Bie
2019-04-08  8:44 ` [dpdk-dev] [RFC 0/3] Add mdev (Mediated device) support in DPDK Alejandro Lucero
2019-04-08  9:36   ` Tiwei Bie
2019-04-10 10:02     ` Francois Ozog
2023-07-03 23:54       ` Stephen Hemminger
2019-07-15  7:52 ` [dpdk-dev] [RFC v2 0/5] " Tiwei Bie
2019-07-15  7:52   ` [dpdk-dev] [RFC v2 1/5] bus/pci: introduce an internal representation of PCI device Tiwei Bie
2019-07-15  7:52   ` [dpdk-dev] [RFC v2 2/5] bus/pci: avoid depending on private value in kernel source Tiwei Bie
2019-07-15  7:52   ` [dpdk-dev] [RFC v2 3/5] bus/pci: introduce helper for MMIO read and write Tiwei Bie
2019-07-15  7:52   ` [dpdk-dev] [RFC v2 4/5] eal: add a helper for reading string from sysfs Tiwei Bie
2019-07-15  7:52   ` [dpdk-dev] [RFC v2 5/5] bus/pci: add mdev support Tiwei Bie
2021-06-01  3:06     ` [dpdk-dev] [RFC v3 0/6] Add mdev (Mediated device) support in DPDK Chenbo Xia
2021-06-01  3:06       ` [dpdk-dev] [RFC v3 1/6] bus/pci: introduce an internal representation of PCI device Chenbo Xia
2021-06-01  3:06       ` [dpdk-dev] [RFC v3 2/6] bus/pci: avoid depending on private value in kernel source Chenbo Xia
2021-06-01  3:06       ` Chenbo Xia [this message]
2021-06-01  3:06       ` [dpdk-dev] [RFC v3 4/6] eal: add a helper for reading string from sysfs Chenbo Xia
2021-06-01  5:37         ` Stephen Hemminger
2021-06-08  5:47           ` Xia, Chenbo
2021-06-01  5:39         ` Stephen Hemminger
2021-06-08  5:48           ` Xia, Chenbo
2021-06-11  7:19         ` Thomas Monjalon
2021-06-01  3:06       ` [dpdk-dev] [RFC v3 5/6] bus/pci: add mdev support Chenbo Xia
2021-06-01  3:06       ` [dpdk-dev] [RFC v3 6/6] bus/pci: add sparse mmap support for mediated PCI devices Chenbo Xia
2021-06-11  7:15       ` [dpdk-dev] [RFC v3 0/6] Add mdev (Mediated device) support in DPDK Thomas Monjalon
2021-06-15  2:49         ` Xia, Chenbo
2021-06-15  7:48           ` Thomas Monjalon
2021-06-15 10:44             ` Xia, Chenbo
2021-06-15 11:57             ` Jason Gunthorpe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210601030644.3318-4-chenbo.xia@intel.com \
    --to=chenbo.xia@intel.com \
    --cc=anatoly.burakov@intel.com \
    --cc=bruce.richardson@intel.com \
    --cc=cunming.liang@intel.com \
    --cc=david.marchand@redhat.com \
    --cc=dev@dpdk.org \
    --cc=ferruh.yigit@intel.com \
    --cc=jingjing.wu@intel.com \
    --cc=konstantin.ananyev@intel.com \
    --cc=mdr@ashroe.eu \
    --cc=nhorman@tuxdriver.com \
    --cc=stephen@networkplumber.org \
    --cc=thomas@monjalon.net \
    --cc=tiwei.bie@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.