[PATCH v6] app/testpmd: add IFPGA AFU register access function

[Rosen Xu <rosen.xu@intel.com>]

Currently register read/write of testpmd is only for PCI device,
but more and more IFPGA based AFU devices need this feature to
access registers, this patch will add support for it.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>

v5 updates:
===========
 - Added Macro to fix compile dependency of ifpga for testpmd
---
 app/test-pmd/config.c  | 253 ++++++++++++++++++++++++++++++++++++++++++++++++-
 app/test-pmd/testpmd.h |  64 +++++++++++++
 2 files changed, 315 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index b9e5dd9..5600ef5 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -866,7 +866,50 @@ void print_valid_ports(void)
 	printf("Invalid vlan_id %d (must be < 4096)\n", vlan_id);
 	return 1;
 }
+#ifdef RTE_LIBRTE_IFPGA_BUS
+static int
+port_reg_off_is_invalid(portid_t port_id, uint32_t reg_off)
+{
+	const struct rte_pci_device *pci_dev;
+	const struct rte_bus *bus;
+	uint64_t len;
+	const struct rte_afu_device *afu_dev;
 
+	if (reg_off & 0x3) {
+		printf("Port register offset 0x%X not aligned on a 4-byte "
+		       "boundary\n",
+		       (unsigned int)reg_off);
+		return 1;
+	}
+
+	if (!ports[port_id].dev_info.device) {
+		printf("Invalid device\n");
+		return 0;
+	}
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		pci_dev = RTE_DEV_TO_PCI(ports[port_id].dev_info.device);
+		len = pci_dev->mem_resource[0].len;
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		afu_dev = RTE_DEV_TO_AFU(ports[port_id].dev_info.device);
+		len = afu_dev->mem_resource[0].len;
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return 1;
+	}
+
+	if (reg_off >= len) {
+		printf("Port %d: register offset %u (0x%X) out of port "
+		       "PCI or AFU device "
+		       "resource (length=%"PRIu64")\n",
+		       port_id, (unsigned int)reg_off,
+			(unsigned int)reg_off, len);
+		return 1;
+	}
+	return 0;
+}
+#else
 static int
 port_reg_off_is_invalid(portid_t port_id, uint32_t reg_off)
 {
@@ -903,7 +946,7 @@ void print_valid_ports(void)
 	}
 	return 0;
 }
-
+#endif
 static int
 reg_bit_pos_is_invalid(uint8_t bit_pos)
 {
@@ -923,6 +966,212 @@ void print_valid_ports(void)
 	printf("0x%08X (%u)\n", (unsigned)reg_v, (unsigned)reg_v);
 }
 
+#ifdef RTE_LIBRTE_IFPGA_BUS
+void
+port_reg_bit_display(portid_t port_id, uint32_t reg_off, uint8_t bit_x)
+{
+	uint32_t reg_v;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit_x))
+		return;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_and_reg_off(port_id, (unsigned int)reg_off);
+	printf("bit %d=%d\n", bit_x, (int) ((reg_v & (1 << bit_x)) >> bit_x));
+}
+
+void
+port_reg_bit_field_display(portid_t port_id, uint32_t reg_off,
+			   uint8_t bit1_pos, uint8_t bit2_pos)
+{
+	uint32_t reg_v;
+	uint8_t  l_bit;
+	uint8_t  h_bit;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit1_pos))
+		return;
+	if (reg_bit_pos_is_invalid(bit2_pos))
+		return;
+	if (bit1_pos > bit2_pos)
+		l_bit = bit2_pos, h_bit = bit1_pos;
+	else
+		l_bit = bit1_pos, h_bit = bit2_pos;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	reg_v >>= l_bit;
+	if (h_bit < 31)
+		reg_v &= ((1 << (h_bit - l_bit + 1)) - 1);
+	display_port_and_reg_off(port_id, (unsigned int)reg_off);
+	printf("bits[%d, %d]=0x%0*X (%u)\n", l_bit, h_bit,
+	       ((h_bit - l_bit) / 4) + 1, (unsigned int)reg_v,
+		(unsigned int)reg_v);
+}
+
+void
+port_reg_display(portid_t port_id, uint32_t reg_off)
+{
+	uint32_t reg_v;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+
+void
+port_reg_bit_set(portid_t port_id, uint32_t reg_off, uint8_t bit_pos,
+		 uint8_t bit_v)
+{
+	uint32_t reg_v;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit_pos))
+		return;
+	if (bit_v > 1) {
+		printf("Invalid bit value %d (must be 0 or 1)\n", (int) bit_v);
+		return;
+	}
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+		if (bit_v == 0)
+			reg_v &= ~(1 << bit_pos);
+		else
+			reg_v |= (1 << bit_pos);
+		port_id_pci_reg_write(port_id, reg_off, reg_v);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+		if (bit_v == 0)
+			reg_v &= ~(1 << bit_pos);
+		else
+			reg_v |= (1 << bit_pos);
+		port_id_afu_reg_write(port_id, reg_off, reg_v);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+
+void
+port_reg_bit_field_set(portid_t port_id, uint32_t reg_off,
+		       uint8_t bit1_pos, uint8_t bit2_pos, uint32_t value)
+{
+	uint32_t max_v;
+	uint32_t reg_v;
+	uint8_t  l_bit;
+	uint8_t  h_bit;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit1_pos))
+		return;
+	if (reg_bit_pos_is_invalid(bit2_pos))
+		return;
+	if (bit1_pos > bit2_pos)
+		l_bit = bit2_pos, h_bit = bit1_pos;
+	else
+		l_bit = bit1_pos, h_bit = bit2_pos;
+
+	if ((h_bit - l_bit) < 31)
+		max_v = (1 << (h_bit - l_bit + 1)) - 1;
+	else
+		max_v = 0xFFFFFFFF;
+
+	if (value > max_v) {
+		printf("Invalid value %u (0x%x) must be < %u (0x%x)\n",
+				(unsigned int)value, (unsigned int)value,
+				(unsigned int)max_v, (unsigned int)max_v);
+		return;
+	}
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+		reg_v &= ~(max_v << l_bit); /* Keep unchanged bits */
+		reg_v |= (value << l_bit); /* Set changed bits */
+		port_id_pci_reg_write(port_id, reg_off, reg_v);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+		reg_v &= ~(max_v << l_bit); /* Keep unchanged bits */
+		reg_v |= (value << l_bit); /* Set changed bits */
+		port_id_afu_reg_write(port_id, reg_off, reg_v);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+
+void
+port_reg_set(portid_t port_id, uint32_t reg_off, uint32_t reg_v)
+{
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		port_id_pci_reg_write(port_id, reg_off, reg_v);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		port_id_afu_reg_write(port_id, reg_off, reg_v);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+#else
 void
 port_reg_bit_display(portid_t port_id, uint32_t reg_off, uint8_t bit_x)
 {
@@ -1058,7 +1307,7 @@ void print_valid_ports(void)
 	port_id_pci_reg_write(port_id, reg_off, reg_v);
 	display_port_reg_value(port_id, reg_off, reg_v);
 }
-
+#endif
 void
 port_mtu_set(portid_t port_id, uint16_t mtu)
 {
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 3ff11e6..49b5de5 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -11,6 +11,9 @@
 #include <rte_bus_pci.h>
 #include <rte_gro.h>
 #include <rte_gso.h>
+#ifdef RTE_LIBRTE_IFPGA_BUS
+#include <rte_bus_ifpga.h>
+#endif
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -671,6 +674,67 @@ struct mplsoudp_decap_conf {
 #define port_id_pci_reg_write(pt_id, reg_off, reg_value) \
 	port_pci_reg_write(&ports[(pt_id)], (reg_off), (reg_value))
 
+#ifdef RTE_LIBRTE_IFPGA_BUS
+/**
+ * Read/Write operations on an AFU register of a port.
+ */
+static inline uint32_t
+port_afu_reg_read(struct rte_port *port, uint32_t reg_off)
+{
+	const struct rte_afu_device *afu_dev;
+	const struct rte_bus *bus;
+	void *reg_addr;
+	uint32_t reg_v;
+
+	if (!port->dev_info.device) {
+		printf("Invalid device\n");
+		return 0;
+	}
+
+	bus = rte_bus_find_by_device(port->dev_info.device);
+	if (bus && !strcmp(bus->name, "ifpga")) {
+		afu_dev = RTE_DEV_TO_AFU(port->dev_info.device);
+	} else {
+		printf("Not an IFPGA AFU device\n");
+		return 0;
+	}
+
+	reg_addr = ((char *)afu_dev->mem_resource[0].addr + reg_off);
+	reg_v = *((volatile uint32_t *)reg_addr);
+	return rte_le_to_cpu_32(reg_v);
+}
+
+#define port_id_afu_reg_read(pt_id, reg_off) \
+	port_afu_reg_read(&ports[(pt_id)], (reg_off))
+
+static inline void
+port_afu_reg_write(struct rte_port *port, uint32_t reg_off, uint32_t reg_v)
+{
+	const struct rte_afu_device *afu_dev;
+	const struct rte_bus *bus;
+	void *reg_addr;
+
+	if (!port->dev_info.device) {
+		printf("Invalid device\n");
+		return;
+	}
+
+	bus = rte_bus_find_by_device(port->dev_info.device);
+	if (bus && !strcmp(bus->name, "ifpga")) {
+		afu_dev = RTE_DEV_TO_AFU(port->dev_info.device);
+	} else {
+		printf("Not an IFPGA AFU device\n");
+		return;
+	}
+
+	reg_addr = ((char *)afu_dev->mem_resource[0].addr + reg_off);
+	*((volatile uint32_t *)reg_addr) = rte_cpu_to_le_32(reg_v);
+}
+
+#define port_id_afu_reg_write(pt_id, reg_off, reg_value) \
+	port_afu_reg_write(&ports[(pt_id)], (reg_off), (reg_value))
+#endif
+
 /* Prototypes */
 unsigned int parse_item_list(char* str, const char* item_name,
 			unsigned int max_items,
-- 
1.8.3.1