[RFC v2 1/9] net/gve: add Tx queue setup for DQO

[Junfeng Guo <junfeng.guo@intel.com>]

Add support for tx_queue_setup_dqo ops.

DQO format has submission and completion queue pair for each Tx/Rx
queue. Note that with DQO format all descriptors and doorbells, as
well as counters are written in little-endian.

Signed-off-by: Junfeng Guo <junfeng.guo@intel.com>
Signed-off-by: Rushil Gupta <rushilg@google.com>
Signed-off-by: Jordan Kimbrough <jrkim@google.com>
Signed-off-by: Jeroen de Borst <jeroendb@google.com>
---
 .mailmap                            |   3 +
 MAINTAINERS                         |   3 +
 drivers/net/gve/base/gve.h          |   1 +
 drivers/net/gve/base/gve_desc_dqo.h |   4 -
 drivers/net/gve/base/gve_osdep.h    |   4 +
 drivers/net/gve/gve_ethdev.c        |  16 ++-
 drivers/net/gve/gve_ethdev.h        |  33 +++++-
 drivers/net/gve/gve_tx_dqo.c        | 178 ++++++++++++++++++++++++++++
 drivers/net/gve/meson.build         |   1 +
 9 files changed, 235 insertions(+), 8 deletions(-)
 create mode 100644 drivers/net/gve/gve_tx_dqo.c

diff --git a/.mailmap b/.mailmap
index 452267a567..553b9ce3ca 100644
--- a/.mailmap
+++ b/.mailmap
@@ -578,6 +578,7 @@ Jens Freimann <jfreimann@redhat.com> <jfreiman@redhat.com>
 Jeremy Plsek <jplsek@iol.unh.edu>
 Jeremy Spewock <jspewock@iol.unh.edu>
 Jerin Jacob <jerinj@marvell.com> <jerin.jacob@caviumnetworks.com> <jerinjacobk@gmail.com>
+Jeroen de Borst <jeroendb@google.com>
 Jerome Jutteau <jerome.jutteau@outscale.com>
 Jerry Hao OS <jerryhao@os.amperecomputing.com>
 Jerry Lilijun <jerry.lilijun@huawei.com>
@@ -642,6 +643,7 @@ Jonathan Erb <jonathan.erb@banduracyber.com>
 Jon DeVree <nuxi@vault24.org>
 Jon Loeliger <jdl@netgate.com>
 Joongi Kim <joongi@an.kaist.ac.kr>
+Jordan Kimbrough <jrkim@google.com>
 Jørgen Østergaard Sloth <jorgen.sloth@xci.dk>
 Jörg Thalheim <joerg@thalheim.io>
 Joseph Richard <joseph.richard@windriver.com>
@@ -1145,6 +1147,7 @@ Roy Franz <roy.franz@cavium.com>
 Roy Pledge <roy.pledge@nxp.com>
 Roy Shterman <roy.shterman@vastdata.com>
 Ruifeng Wang <ruifeng.wang@arm.com>
+Rushil Gupta <rushilg@google.com>
 Ryan E Hall <ryan.e.hall@intel.com>
 Sabyasachi Sengupta <sabyasg@hpe.com>
 Sachin Saxena <sachin.saxena@nxp.com> <sachin.saxena@oss.nxp.com>
diff --git a/MAINTAINERS b/MAINTAINERS
index 9a0f416d2e..7ffa709b3b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -703,6 +703,9 @@ F: doc/guides/nics/features/enic.ini
 
 Google Virtual Ethernet
 M: Junfeng Guo <junfeng.guo@intel.com>
+M: Jeroen de Borst <jeroendb@google.com>
+M: Rushil Gupta <rushilg@google.com>
+M: Jordan Kimbrough <jrkim@google.com>
 F: drivers/net/gve/
 F: doc/guides/nics/gve.rst
 F: doc/guides/nics/features/gve.ini
diff --git a/drivers/net/gve/base/gve.h b/drivers/net/gve/base/gve.h
index 2dc4507acb..2b7cf7d99b 100644
--- a/drivers/net/gve/base/gve.h
+++ b/drivers/net/gve/base/gve.h
@@ -7,6 +7,7 @@
 #define _GVE_H_
 
 #include "gve_desc.h"
+#include "gve_desc_dqo.h"
 
 #define GVE_VERSION		"1.3.0"
 #define GVE_VERSION_PREFIX	"GVE-"
diff --git a/drivers/net/gve/base/gve_desc_dqo.h b/drivers/net/gve/base/gve_desc_dqo.h
index ee1afdecb8..bb4a18d4d1 100644
--- a/drivers/net/gve/base/gve_desc_dqo.h
+++ b/drivers/net/gve/base/gve_desc_dqo.h
@@ -13,10 +13,6 @@
 #define GVE_TX_MAX_HDR_SIZE_DQO 255
 #define GVE_TX_MIN_TSO_MSS_DQO 88
 
-#ifndef __LITTLE_ENDIAN_BITFIELD
-#error "Only little endian supported"
-#endif
-
 /* Basic TX descriptor (DTYPE 0x0C) */
 struct gve_tx_pkt_desc_dqo {
 	__le64 buf_addr;
diff --git a/drivers/net/gve/base/gve_osdep.h b/drivers/net/gve/base/gve_osdep.h
index 7cb73002f4..abf3d379ae 100644
--- a/drivers/net/gve/base/gve_osdep.h
+++ b/drivers/net/gve/base/gve_osdep.h
@@ -35,6 +35,10 @@ typedef rte_be16_t __be16;
 typedef rte_be32_t __be32;
 typedef rte_be64_t __be64;
 
+typedef rte_le16_t __le16;
+typedef rte_le32_t __le32;
+typedef rte_le64_t __le64;
+
 typedef rte_iova_t dma_addr_t;
 
 #define ETH_MIN_MTU	RTE_ETHER_MIN_MTU
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c
index 97781f0ed3..d03f2fba92 100644
--- a/drivers/net/gve/gve_ethdev.c
+++ b/drivers/net/gve/gve_ethdev.c
@@ -299,6 +299,7 @@ gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 
 	dev_info->default_txconf = (struct rte_eth_txconf) {
 		.tx_free_thresh = GVE_DEFAULT_TX_FREE_THRESH,
+		.tx_rs_thresh = GVE_DEFAULT_TX_RS_THRESH,
 		.offloads = 0,
 	};
 
@@ -360,6 +361,13 @@ static const struct eth_dev_ops gve_eth_dev_ops = {
 	.mtu_set              = gve_dev_mtu_set,
 };
 
+static void
+gve_eth_dev_ops_override(struct eth_dev_ops *local_eth_dev_ops)
+{
+	/* override eth_dev ops for DQO */
+	local_eth_dev_ops->tx_queue_setup = gve_tx_queue_setup_dqo;
+}
+
 static void
 gve_free_counter_array(struct gve_priv *priv)
 {
@@ -595,6 +603,7 @@ gve_teardown_priv_resources(struct gve_priv *priv)
 static int
 gve_dev_init(struct rte_eth_dev *eth_dev)
 {
+	static struct eth_dev_ops gve_local_eth_dev_ops = gve_eth_dev_ops;
 	struct gve_priv *priv = eth_dev->data->dev_private;
 	int max_tx_queues, max_rx_queues;
 	struct rte_pci_device *pci_dev;
@@ -602,8 +611,6 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
 	rte_be32_t *db_bar;
 	int err;
 
-	eth_dev->dev_ops = &gve_eth_dev_ops;
-
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
@@ -642,9 +649,12 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
 		eth_dev->rx_pkt_burst = gve_rx_burst;
 		eth_dev->tx_pkt_burst = gve_tx_burst;
 	} else {
-		PMD_DRV_LOG(ERR, "DQO_RDA is not implemented and will be added in the future");
+		/* override Tx/Rx setup/release eth_dev ops */
+		gve_eth_dev_ops_override(&gve_local_eth_dev_ops);
 	}
 
+	eth_dev->dev_ops = &gve_local_eth_dev_ops;
+
 	eth_dev->data->mac_addrs = &priv->dev_addr;
 
 	return 0;
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h
index 235e55899e..2dfcef6893 100644
--- a/drivers/net/gve/gve_ethdev.h
+++ b/drivers/net/gve/gve_ethdev.h
@@ -11,6 +11,9 @@
 
 #include "base/gve.h"
 
+/* TODO: this is a workaround to ensure that Tx complq is enough */
+#define DQO_TX_MULTIPLIER 4
+
 /*
  * Following macros are derived from linux/pci_regs.h, however,
  * we can't simply include that header here, as there is no such
@@ -25,7 +28,8 @@
 #define PCI_MSIX_FLAGS_QSIZE	0x07FF	/* Table size */
 
 #define GVE_DEFAULT_RX_FREE_THRESH  512
-#define GVE_DEFAULT_TX_FREE_THRESH  256
+#define GVE_DEFAULT_TX_FREE_THRESH   32
+#define GVE_DEFAULT_TX_RS_THRESH     32
 #define GVE_TX_MAX_FREE_SZ          512
 
 #define GVE_MIN_BUF_SIZE	    1024
@@ -50,6 +54,13 @@ union gve_tx_desc {
 	struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
 };
 
+/* Tx desc for DQO format */
+union gve_tx_desc_dqo {
+	struct gve_tx_pkt_desc_dqo pkt;
+	struct gve_tx_tso_context_desc_dqo tso_ctx;
+	struct gve_tx_general_context_desc_dqo general_ctx;
+};
+
 /* Offload features */
 union gve_tx_offload {
 	uint64_t data;
@@ -78,8 +89,10 @@ struct gve_tx_queue {
 	uint32_t tx_tail;
 	uint16_t nb_tx_desc;
 	uint16_t nb_free;
+	uint16_t nb_used;
 	uint32_t next_to_clean;
 	uint16_t free_thresh;
+	uint16_t rs_thresh;
 
 	/* Only valid for DQO_QPL queue format */
 	uint16_t sw_tail;
@@ -102,6 +115,17 @@ struct gve_tx_queue {
 	const struct rte_memzone *qres_mz;
 	struct gve_queue_resources *qres;
 
+	/* newly added for DQO*/
+	volatile union gve_tx_desc_dqo *tx_ring;
+	struct gve_tx_compl_desc *compl_ring;
+	const struct rte_memzone *compl_ring_mz;
+	uint64_t compl_ring_phys_addr;
+	uint32_t complq_tail;
+	uint16_t sw_size;
+	uint8_t cur_gen_bit;
+	uint32_t last_desc_cleaned;
+	void **txqs;
+
 	/* Only valid for DQO_RDA queue format */
 	struct gve_tx_queue *complq;
 
@@ -308,4 +332,11 @@ gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 uint16_t
 gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
 
+/* Below functions are used for DQO */
+
+int
+gve_tx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
+		       uint16_t nb_desc, unsigned int socket_id,
+		       const struct rte_eth_txconf *conf);
+
 #endif /* _GVE_ETHDEV_H_ */
diff --git a/drivers/net/gve/gve_tx_dqo.c b/drivers/net/gve/gve_tx_dqo.c
new file mode 100644
index 0000000000..4f8bad31bb
--- /dev/null
+++ b/drivers/net/gve/gve_tx_dqo.c
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Intel Corporation
+ */
+
+#include "gve_ethdev.h"
+#include "base/gve_adminq.h"
+
+static int
+check_tx_thresh_dqo(uint16_t nb_desc, uint16_t tx_rs_thresh,
+		    uint16_t tx_free_thresh)
+{
+	if (tx_rs_thresh >= (nb_desc - 2)) {
+		PMD_DRV_LOG(ERR, "tx_rs_thresh (%u) must be less than the "
+			    "number of TX descriptors (%u) minus 2",
+			    tx_rs_thresh, nb_desc);
+		return -EINVAL;
+	}
+	if (tx_free_thresh >= (nb_desc - 3)) {
+		PMD_DRV_LOG(ERR, "tx_free_thresh (%u) must be less than the "
+			    "number of TX descriptors (%u) minus 3.",
+			    tx_free_thresh, nb_desc);
+		return -EINVAL;
+	}
+	if (tx_rs_thresh > tx_free_thresh) {
+		PMD_DRV_LOG(ERR, "tx_rs_thresh (%u) must be less than or "
+			    "equal to tx_free_thresh (%u).",
+			    tx_rs_thresh, tx_free_thresh);
+		return -EINVAL;
+	}
+	if ((nb_desc % tx_rs_thresh) != 0) {
+		PMD_DRV_LOG(ERR, "tx_rs_thresh (%u) must be a divisor of the "
+			    "number of TX descriptors (%u).",
+			    tx_rs_thresh, nb_desc);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void
+gve_reset_txq_dqo(struct gve_tx_queue *txq)
+{
+	struct rte_mbuf **sw_ring;
+	uint32_t size, i;
+
+	if (txq == NULL) {
+		PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
+		return;
+	}
+
+	size = txq->nb_tx_desc * sizeof(union gve_tx_desc_dqo);
+	for (i = 0; i < size; i++)
+		((volatile char *)txq->tx_ring)[i] = 0;
+
+	size = txq->sw_size * sizeof(struct gve_tx_compl_desc);
+	for (i = 0; i < size; i++)
+		((volatile char *)txq->compl_ring)[i] = 0;
+
+	sw_ring = txq->sw_ring;
+	for (i = 0; i < txq->sw_size; i++)
+		sw_ring[i] = NULL;
+
+	txq->tx_tail = 0;
+	txq->nb_used = 0;
+
+	txq->last_desc_cleaned = 0;
+	txq->sw_tail = 0;
+	txq->nb_free = txq->nb_tx_desc - 1;
+
+	txq->complq_tail = 0;
+	txq->cur_gen_bit = 1;
+}
+
+int
+gve_tx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
+		       uint16_t nb_desc, unsigned int socket_id,
+		       const struct rte_eth_txconf *conf)
+{
+	struct gve_priv *hw = dev->data->dev_private;
+	const struct rte_memzone *mz;
+	struct gve_tx_queue *txq;
+	uint16_t free_thresh;
+	uint16_t rs_thresh;
+	uint16_t sw_size;
+	int err = 0;
+
+	if (nb_desc != hw->tx_desc_cnt) {
+		PMD_DRV_LOG(WARNING, "gve doesn't support nb_desc config, use hw nb_desc %u.",
+			    hw->tx_desc_cnt);
+	}
+	nb_desc = hw->tx_desc_cnt;
+
+	/* Allocate the TX queue data structure. */
+	txq = rte_zmalloc_socket("gve txq",
+				 sizeof(struct gve_tx_queue),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (txq == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to allocate memory for tx queue structure");
+		return -ENOMEM;
+	}
+
+	/* need to check free_thresh here */
+	free_thresh = conf->tx_free_thresh ?
+			conf->tx_free_thresh : GVE_DEFAULT_TX_FREE_THRESH;
+	rs_thresh = conf->tx_rs_thresh ?
+			conf->tx_rs_thresh : GVE_DEFAULT_TX_RS_THRESH;
+	if (check_tx_thresh_dqo(nb_desc, rs_thresh, free_thresh))
+		return -EINVAL;
+
+	txq->nb_tx_desc = nb_desc;
+	txq->free_thresh = free_thresh;
+	txq->rs_thresh = rs_thresh;
+	txq->queue_id = queue_id;
+	txq->port_id = dev->data->port_id;
+	txq->ntfy_id = queue_id;
+	txq->hw = hw;
+	txq->ntfy_addr = &hw->db_bar2[rte_be_to_cpu_32(hw->irq_dbs[txq->ntfy_id].id)];
+
+	/* Allocate software ring */
+	sw_size = nb_desc * DQO_TX_MULTIPLIER;
+	txq->sw_ring = rte_zmalloc_socket("gve tx sw ring",
+					  sw_size * sizeof(struct rte_mbuf *),
+					  RTE_CACHE_LINE_SIZE, socket_id);
+	if (txq->sw_ring == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring");
+		err = -ENOMEM;
+		goto err_txq;
+	}
+	txq->sw_size = sw_size;
+
+	/* Allocate TX hardware ring descriptors. */
+	mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
+				      nb_desc * sizeof(union gve_tx_desc_dqo),
+				      PAGE_SIZE, socket_id);
+	if (mz == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX");
+		err = -ENOMEM;
+		goto err_txq;
+	}
+	txq->tx_ring = (union gve_tx_desc_dqo *)mz->addr;
+	txq->tx_ring_phys_addr = mz->iova;
+	txq->mz = mz;
+
+	/* Allocate TX completion ring descriptors. */
+	mz = rte_eth_dma_zone_reserve(dev, "tx_compl_ring", queue_id,
+				      sw_size * sizeof(struct gve_tx_compl_desc),
+				      PAGE_SIZE, socket_id);
+	if (mz == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX completion queue");
+		err = -ENOMEM;
+		goto err_txq;
+	}
+	txq->compl_ring = (struct gve_tx_compl_desc *)mz->addr;
+	txq->compl_ring_phys_addr = mz->iova;
+	txq->compl_ring_mz = mz;
+	txq->txqs = dev->data->tx_queues;
+
+	mz = rte_eth_dma_zone_reserve(dev, "txq_res", queue_id,
+				      sizeof(struct gve_queue_resources),
+				      PAGE_SIZE, socket_id);
+	if (mz == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX resource");
+		err = -ENOMEM;
+		goto err_txq;
+	}
+	txq->qres = (struct gve_queue_resources *)mz->addr;
+	txq->qres_mz = mz;
+
+	gve_reset_txq_dqo(txq);
+
+	dev->data->tx_queues[queue_id] = txq;
+
+	return 0;
+
+err_txq:
+	rte_free(txq);
+	return err;
+}
diff --git a/drivers/net/gve/meson.build b/drivers/net/gve/meson.build
index af0010c01c..2ddb0cbf9e 100644
--- a/drivers/net/gve/meson.build
+++ b/drivers/net/gve/meson.build
@@ -11,6 +11,7 @@ sources = files(
         'base/gve_adminq.c',
         'gve_rx.c',
         'gve_tx.c',
+        'gve_tx_dqo.c',
         'gve_ethdev.c',
 )
 includes += include_directories('base')
-- 
2.34.1