Linux-RDMA Archive on lore.kernel.org
 help / color / Atom feed
From: Alexander Lobakin <alobakin@marvell.com>
To: "David S. Miller" <davem@davemloft.net>,
	Jakub Kicinski <kuba@kernel.org>
Cc: Alexander Lobakin <alobakin@marvell.com>,
	Igor Russkikh <irusskikh@marvell.com>,
	Michal Kalderon <michal.kalderon@marvell.com>,
	"Ariel Elior" <aelior@marvell.com>,
	Denis Bolotin <denis.bolotin@marvell.com>,
	"Doug Ledford" <dledford@redhat.com>,
	Jason Gunthorpe <jgg@ziepe.ca>,
	"Alexei Starovoitov" <ast@kernel.org>,
	Daniel Borkmann <daniel@iogearbox.net>,
	"Jesper Dangaard Brouer" <hawk@kernel.org>,
	John Fastabend <john.fastabend@gmail.com>,
	Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>,
	"Yonghong Song" <yhs@fb.com>, Andrii Nakryiko <andriin@fb.com>,
	KP Singh <kpsingh@chromium.org>,
	<GR-everest-linux-l2@marvell.com>, <netdev@vger.kernel.org>,
	<bpf@vger.kernel.org>, <linux-rdma@vger.kernel.org>,
	<linux-kernel@vger.kernel.org>
Subject: [PATCH net-next 15/15] qede: add .ndo_xdp_xmit() and XDP_REDIRECT support
Date: Wed, 22 Jul 2020 18:53:49 +0300
Message-ID: <20200722155349.747-16-alobakin@marvell.com> (raw)
In-Reply-To: <20200722155349.747-1-alobakin@marvell.com>

Add XDP_REDIRECT case handling and the corresponding NDO to support
redirecting XDP frames. This also includes registering driver memory
model (currently order-0 page mode) in BPF subsystem.
The total number of XDP queues is usually 1:1 with Rx ones.

Signed-off-by: Alexander Lobakin <alobakin@marvell.com>
Signed-off-by: Igor Russkikh <irusskikh@marvell.com>
Signed-off-by: Michal Kalderon <michal.kalderon@marvell.com>
---
 drivers/net/ethernet/qlogic/qede/qede.h      |  8 ++
 drivers/net/ethernet/qlogic/qede/qede_fp.c   | 97 +++++++++++++++++++-
 drivers/net/ethernet/qlogic/qede/qede_main.c | 18 ++++
 3 files changed, 118 insertions(+), 5 deletions(-)

diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index 308c66a5f98f..803c1fcca8ad 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -199,6 +199,7 @@ struct qede_dev {
 	u8				fp_num_rx;
 	u16				req_queues;
 	u16				num_queues;
+	u16				total_xdp_queues;
 
 #define QEDE_QUEUE_CNT(edev)		((edev)->num_queues)
 #define QEDE_RSS_COUNT(edev)		((edev)->num_queues - (edev)->fp_num_tx)
@@ -381,6 +382,7 @@ struct sw_tx_bd {
 
 struct sw_tx_xdp {
 	struct page			*page;
+	struct xdp_frame		*xdpf;
 	dma_addr_t			mapping;
 };
 
@@ -403,6 +405,9 @@ struct qede_tx_queue {
 	void __iomem			*doorbell_addr;
 	union db_prod			tx_db;
 
+	/* Spinlock for XDP queues in case of XDP_REDIRECT */
+	spinlock_t			xdp_tx_lock;
+
 	int				index; /* Slowpath only */
 #define QEDE_TXQ_XDP_TO_IDX(edev, txq)	((txq)->index - \
 					 QEDE_MAX_TSS_CNT(edev))
@@ -456,6 +461,7 @@ struct qede_fastpath {
 
 	u8				xdp_xmit;
 #define QEDE_XDP_TX			BIT(0)
+#define QEDE_XDP_REDIRECT		BIT(1)
 
 	struct napi_struct		napi;
 	struct qed_sb_info		*sb_info;
@@ -516,6 +522,8 @@ struct qede_reload_args {
 
 /* Datapath functions definition */
 netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev);
+int qede_xdp_transmit(struct net_device *dev, int n_frames,
+		      struct xdp_frame **frames, u32 flags);
 u16 qede_select_queue(struct net_device *dev, struct sk_buff *skb,
 		      struct net_device *sb_dev);
 netdev_features_t qede_features_check(struct sk_buff *skb,
diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c
index c80bf6d37b89..a2494bf85007 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_fp.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c
@@ -303,7 +303,7 @@ static inline void qede_update_tx_producer(struct qede_tx_queue *txq)
 }
 
 static int qede_xdp_xmit(struct qede_tx_queue *txq, dma_addr_t dma, u16 pad,
-			 u16 len, struct page *page)
+			 u16 len, struct page *page, struct xdp_frame *xdpf)
 {
 	struct eth_tx_1st_bd *bd;
 	struct sw_tx_xdp *xdp;
@@ -330,12 +330,66 @@ static int qede_xdp_xmit(struct qede_tx_queue *txq, dma_addr_t dma, u16 pad,
 	xdp = txq->sw_tx_ring.xdp + txq->sw_tx_prod;
 	xdp->mapping = dma;
 	xdp->page = page;
+	xdp->xdpf = xdpf;
 
 	txq->sw_tx_prod = (txq->sw_tx_prod + 1) % txq->num_tx_buffers;
 
 	return 0;
 }
 
+int qede_xdp_transmit(struct net_device *dev, int n_frames,
+		      struct xdp_frame **frames, u32 flags)
+{
+	struct qede_dev *edev = netdev_priv(dev);
+	struct device *dmadev = &edev->pdev->dev;
+	struct qede_tx_queue *xdp_tx;
+	struct xdp_frame *xdpf;
+	dma_addr_t mapping;
+	int i, drops = 0;
+	u16 xdp_prod;
+
+	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+		return -EINVAL;
+
+	if (unlikely(!netif_running(dev)))
+		return -ENETDOWN;
+
+	i = smp_processor_id() % edev->total_xdp_queues;
+	xdp_tx = edev->fp_array[i].xdp_tx;
+
+	spin_lock(&xdp_tx->xdp_tx_lock);
+
+	for (i = 0; i < n_frames; i++) {
+		xdpf = frames[i];
+
+		mapping = dma_map_single(dmadev, xdpf->data, xdpf->len,
+					 DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(dmadev, mapping))) {
+			xdp_return_frame_rx_napi(xdpf);
+			drops++;
+
+			continue;
+		}
+
+		if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len,
+					   NULL, xdpf))) {
+			xdp_return_frame_rx_napi(xdpf);
+			drops++;
+		}
+	}
+
+	if (flags & XDP_XMIT_FLUSH) {
+		xdp_prod = qed_chain_get_prod_idx(&xdp_tx->tx_pbl);
+
+		xdp_tx->tx_db.data.bd_prod = cpu_to_le16(xdp_prod);
+		qede_update_tx_producer(xdp_tx);
+	}
+
+	spin_unlock(&xdp_tx->xdp_tx_lock);
+
+	return n_frames - drops;
+}
+
 int qede_txq_has_work(struct qede_tx_queue *txq)
 {
 	u16 hw_bd_cons;
@@ -353,6 +407,7 @@ static void qede_xdp_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq)
 {
 	struct sw_tx_xdp *xdp_info, *xdp_arr = txq->sw_tx_ring.xdp;
 	struct device *dev = &edev->pdev->dev;
+	struct xdp_frame *xdpf;
 	u16 hw_bd_cons;
 
 	hw_bd_cons = le16_to_cpu(*txq->hw_cons_ptr);
@@ -360,10 +415,19 @@ static void qede_xdp_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq)
 
 	while (hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl)) {
 		xdp_info = xdp_arr + txq->sw_tx_cons;
+		xdpf = xdp_info->xdpf;
+
+		if (xdpf) {
+			dma_unmap_single(dev, xdp_info->mapping, xdpf->len,
+					 DMA_TO_DEVICE);
+			xdp_return_frame(xdpf);
 
-		dma_unmap_page(dev, xdp_info->mapping, PAGE_SIZE,
-			       DMA_BIDIRECTIONAL);
-		__free_page(xdp_info->page);
+			xdp_info->xdpf = NULL;
+		} else {
+			dma_unmap_page(dev, xdp_info->mapping, PAGE_SIZE,
+				       DMA_BIDIRECTIONAL);
+			__free_page(xdp_info->page);
+		}
 
 		qed_chain_consume(&txq->tx_pbl);
 		txq->sw_tx_cons = (txq->sw_tx_cons + 1) % txq->num_tx_buffers;
@@ -1065,7 +1129,8 @@ static bool qede_rx_xdp(struct qede_dev *edev,
 		 * throw current buffer, as replacement was already allocated.
 		 */
 		if (unlikely(qede_xdp_xmit(fp->xdp_tx, bd->mapping,
-					   *data_offset, *len, bd->data))) {
+					   *data_offset, *len, bd->data,
+					   NULL))) {
 			dma_unmap_page(rxq->dev, bd->mapping, PAGE_SIZE,
 				       rxq->data_direction);
 			__free_page(bd->data);
@@ -1079,6 +1144,25 @@ static bool qede_rx_xdp(struct qede_dev *edev,
 		}
 
 		/* Regardless, we've consumed an Rx BD */
+		qede_rx_bd_ring_consume(rxq);
+		break;
+	case XDP_REDIRECT:
+		/* We need the replacement buffer before transmit. */
+		if (unlikely(qede_alloc_rx_buffer(rxq, true))) {
+			qede_recycle_rx_bd_ring(rxq, 1);
+
+			trace_xdp_exception(edev->ndev, prog, act);
+			break;
+		}
+
+		dma_unmap_page(rxq->dev, bd->mapping, PAGE_SIZE,
+			       rxq->data_direction);
+
+		if (unlikely(xdp_do_redirect(edev->ndev, &xdp, prog)))
+			DP_NOTICE(edev, "Failed to redirect the packet\n");
+		else
+			fp->xdp_xmit |= QEDE_XDP_REDIRECT;
+
 		qede_rx_bd_ring_consume(rxq);
 		break;
 	default:
@@ -1387,6 +1471,9 @@ int qede_poll(struct napi_struct *napi, int budget)
 		qede_update_tx_producer(fp->xdp_tx);
 	}
 
+	if (fp->xdp_xmit & QEDE_XDP_REDIRECT)
+		xdp_do_flush_map();
+
 	return rx_work_done;
 }
 
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index 92bcdfa27961..1aaae3203f5a 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -672,6 +672,7 @@ static const struct net_device_ops qede_netdev_ops = {
 #ifdef CONFIG_RFS_ACCEL
 	.ndo_rx_flow_steer	= qede_rx_flow_steer,
 #endif
+	.ndo_xdp_xmit		= qede_xdp_transmit,
 	.ndo_setup_tc		= qede_setup_tc_offload,
 };
 
@@ -712,6 +713,7 @@ static const struct net_device_ops qede_netdev_vf_xdp_ops = {
 	.ndo_udp_tunnel_del	= udp_tunnel_nic_del_port,
 	.ndo_features_check	= qede_features_check,
 	.ndo_bpf		= qede_xdp,
+	.ndo_xdp_xmit		= qede_xdp_transmit,
 };
 
 /* -------------------------------------------------------------------------
@@ -1712,6 +1714,7 @@ static void qede_init_fp(struct qede_dev *edev)
 {
 	int queue_id, rxq_index = 0, txq_index = 0;
 	struct qede_fastpath *fp;
+	bool init_xdp = false;
 
 	for_each_queue(queue_id) {
 		fp = &edev->fp_array[queue_id];
@@ -1723,6 +1726,9 @@ static void qede_init_fp(struct qede_dev *edev)
 			fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev,
 								rxq_index);
 			fp->xdp_tx->is_xdp = 1;
+
+			spin_lock_init(&fp->xdp_tx->xdp_tx_lock);
+			init_xdp = true;
 		}
 
 		if (fp->type & QEDE_FASTPATH_RX) {
@@ -1738,6 +1744,13 @@ static void qede_init_fp(struct qede_dev *edev)
 			/* Driver have no error path from here */
 			WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
 						 fp->rxq->rxq_id) < 0);
+
+			if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq,
+						       MEM_TYPE_PAGE_ORDER0,
+						       NULL)) {
+				DP_NOTICE(edev,
+					  "Failed to register XDP memory model\n");
+			}
 		}
 
 		if (fp->type & QEDE_FASTPATH_TX) {
@@ -1763,6 +1776,11 @@ static void qede_init_fp(struct qede_dev *edev)
 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
 			 edev->ndev->name, queue_id);
 	}
+
+	if (init_xdp) {
+		edev->total_xdp_queues = QEDE_RSS_COUNT(edev);
+		DP_INFO(edev, "Total XDP queues: %u\n", edev->total_xdp_queues);
+	}
 }
 
 static int qede_set_real_num_queues(struct qede_dev *edev)
-- 
2.25.1


  parent reply index

Thread overview: 17+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-07-22 15:53 [PATCH net-next 00/15] qed/qede: improve chain API and add " Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 01/15] qed: reformat "qed_chain.h" a bit Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 02/15] qed: reformat Makefile Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 03/15] qed: move chain methods to a separate file Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 04/15] qed: prevent possible double-frees of the chains Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 05/15] qed: sanitize PBL chains allocation Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 06/15] qed: move chain initialization inlines next to allocation functions Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 07/15] qed: simplify initialization of the chains with an external PBL Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 08/15] qed: simplify chain allocation with init params struct Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 09/15] qed: add support for different page sizes for chains Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 10/15] qed: optimize common chain accessors Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 11/15] qed: introduce qed_chain_get_elem_used{,u32}() Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 12/15] qede: reformat several structures in "qede.h" Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 13/15] qede: reformat net_device_ops declarations Alexander Lobakin
2020-07-22 15:53 ` [PATCH net-next 14/15] qede: refactor XDP Tx processing Alexander Lobakin
2020-07-22 15:53 ` Alexander Lobakin [this message]
2020-07-22 21:52 ` [PATCH net-next 03/15] qed: move chain methods to a separate file Alexander Lobakin

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200722155349.747-16-alobakin@marvell.com \
    --to=alobakin@marvell.com \
    --cc=GR-everest-linux-l2@marvell.com \
    --cc=aelior@marvell.com \
    --cc=andriin@fb.com \
    --cc=ast@kernel.org \
    --cc=bpf@vger.kernel.org \
    --cc=daniel@iogearbox.net \
    --cc=davem@davemloft.net \
    --cc=denis.bolotin@marvell.com \
    --cc=dledford@redhat.com \
    --cc=hawk@kernel.org \
    --cc=irusskikh@marvell.com \
    --cc=jgg@ziepe.ca \
    --cc=john.fastabend@gmail.com \
    --cc=kafai@fb.com \
    --cc=kpsingh@chromium.org \
    --cc=kuba@kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-rdma@vger.kernel.org \
    --cc=michal.kalderon@marvell.com \
    --cc=netdev@vger.kernel.org \
    --cc=songliubraving@fb.com \
    --cc=yhs@fb.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-RDMA Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-rdma/0 linux-rdma/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-rdma linux-rdma/ https://lore.kernel.org/linux-rdma \
		linux-rdma@vger.kernel.org
	public-inbox-index linux-rdma

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-rdma


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git