[dpdk-dev] [PATCH v3 3/6] net/ark: update internal structs to reflect FPGA updates

[Ed Czeck <ed.czeck@atomicrules.com>]

- New PCIe IDs using net/ark driver
- Update Version IDs and structures specified by hardware
- New internal descriptor status for TX
- Adjust data placement in RX operations, headroom in retained
for segmented mbufs

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
 doc/guides/nics/ark.rst         |   5 ++
 drivers/net/ark/ark_ddm.c       |  18 +++--
 drivers/net/ark/ark_ddm.h       |  22 +++---
 drivers/net/ark/ark_ethdev.c    |   3 +
 drivers/net/ark/ark_ethdev_rx.c |   6 +-
 drivers/net/ark/ark_ethdev_tx.c | 122 +++++++++++++++++++-------------
 drivers/net/ark/ark_udm.c       |   2 +
 drivers/net/ark/ark_udm.h       |  13 ++--
 8 files changed, 121 insertions(+), 70 deletions(-)

diff --git a/doc/guides/nics/ark.rst b/doc/guides/nics/ark.rst
index 18434c7a4..dcccfee26 100644
--- a/doc/guides/nics/ark.rst
+++ b/doc/guides/nics/ark.rst
@@ -155,6 +155,11 @@ ARK PMD supports the following Arkville RTL PCIe instances including:
 
 * ``1d6c:100d`` - AR-ARKA-FX0 [Arkville 32B DPDK Data Mover]
 * ``1d6c:100e`` - AR-ARKA-FX1 [Arkville 64B DPDK Data Mover]
+* ``1d6c:100f`` - AR-ARKA-FX1 [Arkville 64B DPDK Data Mover for Versal]
+* ``1d6c:1010`` - AR-ARKA-FX1 [Arkville 64B DPDK Data Mover for Agilex]
+* ``1d6c:1017`` - AR-ARK-FX1 [Arkville 64B Multi-Homed Primary Endpoint]
+* ``1d6c:1018`` - AR-ARK-FX1 [Arkville 64B Multi-Homed Secondary Endpoint]
+* ``1d6c:1019`` - AR-ARK-FX1 [Arkville 64B Multi-Homed Tertiary Endpoint]
 
 Supported Operating Systems
 ---------------------------
diff --git a/drivers/net/ark/ark_ddm.c b/drivers/net/ark/ark_ddm.c
index 91d1179d8..232137157 100644
--- a/drivers/net/ark/ark_ddm.c
+++ b/drivers/net/ark/ark_ddm.c
@@ -7,6 +7,8 @@
 #include "ark_logs.h"
 #include "ark_ddm.h"
 
+static_assert(sizeof(union ark_tx_meta) == 8, "Unexpected struct size ark_tx_meta");
+
 /* ************************************************************************* */
 int
 ark_ddm_verify(struct ark_ddm_t *ddm)
@@ -19,18 +21,26 @@ ark_ddm_verify(struct ark_ddm_t *ddm)
 	}
 
 	hw_const = ddm->cfg.const0;
+	if (hw_const == ARK_DDM_CONST3)
+		return 0;
+
 	if (hw_const == ARK_DDM_CONST1) {
 		ARK_PMD_LOG(ERR,
 			    "ARK: DDM module is version 1, "
 			    "PMD expects version 2\n");
 		return -1;
-	} else if (hw_const != ARK_DDM_CONST2) {
+	}
+
+	if (hw_const == ARK_DDM_CONST2) {
 		ARK_PMD_LOG(ERR,
-			    "ARK: DDM module not found as expected 0x%08x\n",
-			    ddm->cfg.const0);
+			    "ARK: DDM module is version 2, "
+			    "PMD expects version 3\n");
 		return -1;
 	}
-	return 0;
+	ARK_PMD_LOG(ERR,
+		    "ARK: DDM module not found as expected 0x%08x\n",
+		    ddm->cfg.const0);
+	return -1;
 }
 
 void
diff --git a/drivers/net/ark/ark_ddm.h b/drivers/net/ark/ark_ddm.h
index 5456b4b5c..687ff2519 100644
--- a/drivers/net/ark/ark_ddm.h
+++ b/drivers/net/ark/ark_ddm.h
@@ -16,17 +16,22 @@
  * there is minimal documentation.
  */
 
-/* struct defining Tx meta data --  fixed in FPGA -- 16 bytes */
-struct ark_tx_meta {
+/* struct defining Tx meta data --  fixed in FPGA -- 8 bytes */
+union ark_tx_meta {
 	uint64_t physaddr;
-	uint32_t user1;
-	uint16_t data_len;		/* of this MBUF */
+	struct {
+		uint32_t usermeta0;
+		uint32_t usermeta1;
+	};
+	struct {
+		uint16_t data_len;	/* of this MBUF */
 #define   ARK_DDM_EOP   0x01
 #define   ARK_DDM_SOP   0x02
-	uint8_t flags;		/* bit 0 indicates last mbuf in chain. */
-	uint8_t reserved[1];
-};
-
+		uint8_t  flags;
+		uint8_t  meta_cnt;
+		uint32_t user1;
+	};
+} __rte_packed;
 
 /*
  * DDM core hardware structures
@@ -35,6 +40,7 @@ struct ark_tx_meta {
  */
 #define ARK_DDM_CFG 0x0000
 /* Set unique HW ID for hardware version */
+#define ARK_DDM_CONST3 (0x334d4444)
 #define ARK_DDM_CONST2 (0x324d4444)
 #define ARK_DDM_CONST1 (0xfacecafe)
 
diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 477e1de02..9f0a8d9e8 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -95,6 +95,9 @@ static const char * const valid_arguments[] = {
 static const struct rte_pci_id pci_id_ark_map[] = {
 	{RTE_PCI_DEVICE(0x1d6c, 0x100d)},
 	{RTE_PCI_DEVICE(0x1d6c, 0x100e)},
+	{RTE_PCI_DEVICE(0x1d6c, 0x1017)},
+	{RTE_PCI_DEVICE(0x1d6c, 0x1018)},
+	{RTE_PCI_DEVICE(0x1d6c, 0x1019)},
 	{.vendor_id = 0, /* sentinel */ },
 };
 
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index 8e55b851a..21a9af41a 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -60,7 +60,6 @@ struct ark_rx_queue {
 	volatile uint32_t prod_index;	/* step 2 filled by FPGA */
 } __rte_cache_aligned;
 
-
 /* ************************************************************************* */
 static int
 eth_ark_rx_hw_setup(struct rte_eth_dev *dev,
@@ -265,7 +264,6 @@ eth_ark_recv_pkts(void *rx_queue,
 		/* META DATA embedded in headroom */
 		meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET);
 
-		mbuf->port = meta->port;
 		mbuf->pkt_len = meta->pkt_len;
 		mbuf->data_len = meta->pkt_len;
 		/* set timestamp if enabled at least on one device */
@@ -346,8 +344,7 @@ eth_ark_rx_jumbo(struct ark_rx_queue *queue,
 	/* HW guarantees that the data does not exceed prod_index! */
 	while (remaining != 0) {
 		data_len = RTE_MIN(remaining,
-				   RTE_MBUF_DEFAULT_DATAROOM +
-				   RTE_PKTMBUF_HEADROOM);
+				   RTE_MBUF_DEFAULT_DATAROOM);
 
 		remaining -= data_len;
 		segments += 1;
@@ -356,7 +353,6 @@ eth_ark_rx_jumbo(struct ark_rx_queue *queue,
 		mbuf_prev->next = mbuf;
 		mbuf_prev = mbuf;
 		mbuf->data_len = data_len;
-		mbuf->data_off = 0;
 
 		cons_index += 1;
 	}
diff --git a/drivers/net/ark/ark_ethdev_tx.c b/drivers/net/ark/ark_ethdev_tx.c
index 612d918e3..00e5dbf7c 100644
--- a/drivers/net/ark/ark_ethdev_tx.c
+++ b/drivers/net/ark/ark_ethdev_tx.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
+ * Copyright (c) 2015-2021 Atomic Rules LLC
  */
 
 #include <unistd.h>
@@ -23,7 +23,7 @@
 
 /* ************************************************************************* */
 struct ark_tx_queue {
-	struct ark_tx_meta *meta_q;
+	union ark_tx_meta *meta_q;
 	struct rte_mbuf **bufs;
 
 	/* handles for hw objects */
@@ -37,8 +37,8 @@ struct ark_tx_queue {
 	uint32_t queue_mask;
 
 	/* 3 indexes to the paired data rings. */
-	uint32_t prod_index;		/* where to put the next one */
-	uint32_t free_index;		/* mbuf has been freed */
+	int32_t prod_index;		/* where to put the next one */
+	int32_t free_index;		/* mbuf has been freed */
 
 	/* The queue Id is used to identify the HW Q */
 	uint16_t phys_qid;
@@ -47,14 +47,15 @@ struct ark_tx_queue {
 
 	uint32_t pad[1];
 
-	/* second cache line - fields only used in slow path */
+	/* second cache line - fields written by device */
 	RTE_MARKER cacheline1 __rte_cache_min_aligned;
-	uint32_t cons_index;		/* hw is done, can be freed */
+	volatile int32_t cons_index;		/* hw is done, can be freed */
 } __rte_cache_aligned;
 
 /* Forward declarations */
-static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
-				 struct rte_mbuf *mbuf);
+static int eth_ark_tx_jumbo(struct ark_tx_queue *queue,
+			    struct rte_mbuf *mbuf,
+			    uint32_t *user_meta, uint8_t meta_cnt);
 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
 static void free_completed_tx(struct ark_tx_queue *queue);
 
@@ -66,16 +67,44 @@ ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
 
 /* ************************************************************************* */
 static inline void
-eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
-			  const struct rte_mbuf *mbuf,
-			  uint8_t flags)
+eth_ark_tx_desc_fill(struct ark_tx_queue *queue,
+		     struct rte_mbuf *mbuf,
+		     uint8_t  flags,
+		     uint32_t *user_meta,
+		     uint8_t  meta_cnt /* 0 to 5 */
+		     )
 {
-	meta->physaddr = rte_mbuf_data_iova(mbuf);
-	meta->user1 = rte_pmd_ark_mbuf_tx_userdata_get(mbuf);
+	uint32_t tx_idx;
+	union ark_tx_meta *meta;
+	uint8_t m;
+
+	/* Header */
+	tx_idx = queue->prod_index & queue->queue_mask;
+	meta = &queue->meta_q[tx_idx];
 	meta->data_len = rte_pktmbuf_data_len(mbuf);
 	meta->flags = flags;
+	meta->meta_cnt = meta_cnt / 2;
+	meta->user1 = meta_cnt ? (*user_meta++) : 0;
+	queue->prod_index++;
+
+	queue->bufs[tx_idx] = mbuf;
+
+	/* 1 or 2 user meta data entries, user words 1,2 and 3,4 */
+	for (m = 1; m < meta_cnt; m += 2) {
+		tx_idx = queue->prod_index & queue->queue_mask;
+		meta = &queue->meta_q[tx_idx];
+		meta->usermeta0 = *user_meta++;
+		meta->usermeta1 = *user_meta++;
+		queue->prod_index++;
+	}
+
+	tx_idx = queue->prod_index & queue->queue_mask;
+	meta = &queue->meta_q[tx_idx];
+	meta->physaddr = rte_mbuf_data_iova(mbuf);
+	queue->prod_index++;
 }
 
+
 /* ************************************************************************* */
 uint16_t
 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
@@ -91,12 +120,12 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
 	struct ark_tx_queue *queue;
 	struct rte_mbuf *mbuf;
-	struct ark_tx_meta *meta;
+	uint32_t user_meta;
 
-	uint32_t idx;
-	uint32_t prod_index_limit;
 	int stat;
+	int32_t prod_index_limit;
 	uint16_t nb;
+	uint8_t user_len = 1;
 	const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN;
 
 	queue = (struct ark_tx_queue *)vtxq;
@@ -104,10 +133,11 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	/* free any packets after the HW is done with them */
 	free_completed_tx(queue);
 
-	prod_index_limit = queue->queue_size + queue->free_index;
+	/* leave 4 elements mpu data */
+	prod_index_limit = queue->queue_size + queue->free_index - 4;
 
 	for (nb = 0;
-	     (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
+	     (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0;
 	     ++nb) {
 		mbuf = tx_pkts[nb];
 
@@ -133,19 +163,16 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			memset(appended, 0, to_add);
 		}
 
+		user_meta = rte_pmd_ark_mbuf_tx_userdata_get(mbuf);
 		if (unlikely(mbuf->nb_segs != 1)) {
-			stat = eth_ark_tx_jumbo(queue, mbuf);
+			stat = eth_ark_tx_jumbo(queue, mbuf,
+						&user_meta, user_len);
 			if (unlikely(stat != 0))
 				break;		/* Queue is full */
 		} else {
-			idx = queue->prod_index & queue->queue_mask;
-			queue->bufs[idx] = mbuf;
-			meta = &queue->meta_q[idx];
-			eth_ark_tx_meta_from_mbuf(meta,
-						  mbuf,
-						  ARK_DDM_SOP |
-						  ARK_DDM_EOP);
-			queue->prod_index++;
+			eth_ark_tx_desc_fill(queue, mbuf,
+					     ARK_DDM_SOP | ARK_DDM_EOP,
+					     &user_meta, user_len);
 		}
 	}
 
@@ -173,32 +200,28 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 }
 
 /* ************************************************************************* */
-static uint32_t
-eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
+static int
+eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf,
+		 uint32_t *user_meta, uint8_t meta_cnt)
 {
 	struct rte_mbuf *next;
-	struct ark_tx_meta *meta;
-	uint32_t free_queue_space;
-	uint32_t idx;
+	int32_t free_queue_space;
 	uint8_t flags = ARK_DDM_SOP;
 
 	free_queue_space = queue->queue_mask -
 		(queue->prod_index - queue->free_index);
-	if (unlikely(free_queue_space < mbuf->nb_segs))
+	/* We need up to 4 mbufs for first header and 2 for subsequent ones */
+	if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs))))
 		return -1;
 
 	while (mbuf != NULL) {
 		next = mbuf->next;
-
-		idx = queue->prod_index & queue->queue_mask;
-		queue->bufs[idx] = mbuf;
-		meta = &queue->meta_q[idx];
-
 		flags |= (next == NULL) ? ARK_DDM_EOP : 0;
-		eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
-		queue->prod_index++;
+
+		eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt);
 
 		flags &= ~ARK_DDM_SOP;	/* drop SOP flags */
+		meta_cnt = 0;		/* Meta only on SOP */
 		mbuf = next;
 	}
 
@@ -227,6 +250,9 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	}
 
+	/* Each packet requires at least 2 mpu elements - double desc count */
+	nb_desc = 2 * nb_desc;
+
 	/* Allocate queue struct */
 	queue =	rte_zmalloc_socket("Ark_txqueue",
 				   sizeof(struct ark_tx_queue),
@@ -248,7 +274,7 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
 
 	queue->meta_q =
 		rte_zmalloc_socket("Ark_txqueue meta",
-				   nb_desc * sizeof(struct ark_tx_meta),
+				   nb_desc * sizeof(union ark_tx_meta),
 				   64,
 				   socket_id);
 	queue->bufs =
@@ -289,7 +315,7 @@ eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
 	uint32_t write_interval_ns;
 
 	/* Verify HW -- MPU */
-	if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
+	if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta)))
 		return -1;
 
 	queue_base = rte_malloc_virt2iova(queue);
@@ -391,19 +417,19 @@ static void
 free_completed_tx(struct ark_tx_queue *queue)
 {
 	struct rte_mbuf *mbuf;
-	struct ark_tx_meta *meta;
-	uint32_t top_index;
+	union ark_tx_meta *meta;
+	int32_t top_index;
 
 	top_index = queue->cons_index;	/* read once */
-	while (queue->free_index != top_index) {
+	while ((top_index - queue->free_index) > 0) {
 		meta = &queue->meta_q[queue->free_index & queue->queue_mask];
-		mbuf = queue->bufs[queue->free_index & queue->queue_mask];
-
 		if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
+			mbuf = queue->bufs[queue->free_index &
+					   queue->queue_mask];
 			/* ref count of the mbuf is checked in this call. */
 			rte_pktmbuf_free(mbuf);
 		}
-		queue->free_index++;
+		queue->free_index += (meta->meta_cnt + 2);
 	}
 }
 
diff --git a/drivers/net/ark/ark_udm.c b/drivers/net/ark/ark_udm.c
index a740d36d4..28c4500a2 100644
--- a/drivers/net/ark/ark_udm.c
+++ b/drivers/net/ark/ark_udm.c
@@ -7,6 +7,8 @@
 #include "ark_logs.h"
 #include "ark_udm.h"
 
+static_assert(sizeof(struct ark_rx_meta) == 32, "Unexpected struct size ark_rx_meta");
+
 int
 ark_udm_verify(struct ark_udm_t *udm)
 {
diff --git a/drivers/net/ark/ark_udm.h b/drivers/net/ark/ark_udm.h
index 5846c825b..ea92d4b6e 100644
--- a/drivers/net/ark/ark_udm.h
+++ b/drivers/net/ark/ark_udm.h
@@ -15,14 +15,15 @@
  * there is minimal documentation.
  */
 
-/* Meta data structure apssed from FPGA, must match layout in FPGA */
+/* Meta data structure passed from FPGA, must match layout in FPGA
+ * -- 32 bytes
+ */
 struct ark_rx_meta {
 	uint64_t timestamp;
 	uint64_t user_data;
-	uint8_t port;
-	uint8_t dst_queue;
+	uint8_t  reserved[14];
 	uint16_t pkt_len;
-};
+} __rte_packed;
 
 /*
  * UDM hardware structures
@@ -32,7 +33,9 @@ struct ark_rx_meta {
 
 #define ARK_RX_WRITE_TIME_NS 2500
 #define ARK_UDM_SETUP 0
-#define ARK_UDM_CONST 0xbACECACE
+#define ARK_UDM_CONST2 0xbACECACE
+#define ARK_UDM_CONST3 0x334d4455
+#define ARK_UDM_CONST ARK_UDM_CONST3
 struct ark_udm_setup_t {
 	uint32_t r0;
 	uint32_t r4;
-- 
2.17.1