[PATCH net-next] net: lan966x: Don't use xdp_frame when action is XDP_TX

[PATCH net-next] net: lan966x: Don't use xdp_frame when action is XDP_TX

[Horatiu Vultur]

When the action of an xdp program was XDP_TX, lan966x was creating
a xdp_frame and use this one to send the frame back. But it is also
possible to send back the frame without needing a xdp_frame, because
it possible to send it back using the page.
And then once the frame is transmitted is possible to use directly
page_pool_recycle_direct as lan966x is using page pools.
This would save some CPU usage on this path.

Signed-off-by: Horatiu Vultur <horatiu.vultur@microchip.com>
---
 .../ethernet/microchip/lan966x/lan966x_fdma.c | 35 +++++++++++--------
 .../ethernet/microchip/lan966x/lan966x_main.h |  2 ++
 .../ethernet/microchip/lan966x/lan966x_xdp.c  | 11 +++---
 3 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
index 2ed76bb61a731..7947259e67e4e 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
@@ -390,6 +390,7 @@ static void lan966x_fdma_stop_netdev(struct lan966x *lan966x)
 static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
 {
 	struct lan966x_tx *tx = &lan966x->tx;
+	struct lan966x_rx *rx = &lan966x->rx;
 	struct lan966x_tx_dcb_buf *dcb_buf;
 	struct xdp_frame_bulk bq;
 	struct lan966x_db *db;
@@ -432,7 +433,8 @@ static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
 			if (dcb_buf->xdp_ndo)
 				xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq);
 			else
-				xdp_return_frame_rx_napi(dcb_buf->data.xdpf);
+				page_pool_recycle_direct(rx->page_pool,
+							 dcb_buf->data.page);
 		}
 
 		clear = true;
@@ -702,6 +704,7 @@ static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use)
 int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 			   struct xdp_frame *xdpf,
 			   struct page *page,
+			   u32 len,
 			   bool dma_map)
 {
 	struct lan966x *lan966x = port->lan966x;
@@ -722,6 +725,15 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 		goto out;
 	}
 
+	/* Fill up the buffer */
+	next_dcb_buf = &tx->dcbs_buf[next_to_use];
+	next_dcb_buf->use_skb = false;
+	next_dcb_buf->xdp_ndo = dma_map;
+	next_dcb_buf->len = len + IFH_LEN_BYTES;
+	next_dcb_buf->used = true;
+	next_dcb_buf->ptp = false;
+	next_dcb_buf->dev = port->dev;
+
 	/* Generate new IFH */
 	if (dma_map) {
 		if (xdpf->headroom < IFH_LEN_BYTES) {
@@ -736,16 +748,18 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 
 		dma_addr = dma_map_single(lan966x->dev,
 					  xdpf->data - IFH_LEN_BYTES,
-					  xdpf->len + IFH_LEN_BYTES,
+					  len + IFH_LEN_BYTES,
 					  DMA_TO_DEVICE);
 		if (dma_mapping_error(lan966x->dev, dma_addr)) {
 			ret = NETDEV_TX_OK;
 			goto out;
 		}
 
+		next_dcb_buf->data.xdpf = xdpf;
+
 		/* Setup next dcb */
 		lan966x_fdma_tx_setup_dcb(tx, next_to_use,
-					  xdpf->len + IFH_LEN_BYTES,
+					  len + IFH_LEN_BYTES,
 					  dma_addr);
 	} else {
 		ifh = page_address(page) + XDP_PACKET_HEADROOM;
@@ -756,25 +770,18 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 		dma_addr = page_pool_get_dma_addr(page);
 		dma_sync_single_for_device(lan966x->dev,
 					   dma_addr + XDP_PACKET_HEADROOM,
-					   xdpf->len + IFH_LEN_BYTES,
+					   len + IFH_LEN_BYTES,
 					   DMA_TO_DEVICE);
 
+		next_dcb_buf->data.page = page;
+
 		/* Setup next dcb */
 		lan966x_fdma_tx_setup_dcb(tx, next_to_use,
-					  xdpf->len + IFH_LEN_BYTES,
+					  len + IFH_LEN_BYTES,
 					  dma_addr + XDP_PACKET_HEADROOM);
 	}
 
-	/* Fill up the buffer */
-	next_dcb_buf = &tx->dcbs_buf[next_to_use];
-	next_dcb_buf->use_skb = false;
-	next_dcb_buf->data.xdpf = xdpf;
-	next_dcb_buf->xdp_ndo = dma_map;
-	next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES;
 	next_dcb_buf->dma_addr = dma_addr;
-	next_dcb_buf->used = true;
-	next_dcb_buf->ptp = false;
-	next_dcb_buf->dev = port->dev;
 
 	/* Start the transmission */
 	lan966x_fdma_tx_start(tx, next_to_use);
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
index 851afb0166b19..59da35a2c93d4 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
@@ -243,6 +243,7 @@ struct lan966x_tx_dcb_buf {
 	union {
 		struct sk_buff *skb;
 		struct xdp_frame *xdpf;
+		struct page *page;
 	} data;
 	u32 len;
 	u32 used : 1;
@@ -544,6 +545,7 @@ int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev);
 int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
 			   struct xdp_frame *frame,
 			   struct page *page,
+			   u32 len,
 			   bool dma_map);
 int lan966x_fdma_change_mtu(struct lan966x *lan966x);
 void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev);
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
index 2e6f486ec67d7..a8ad1f4e431cb 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
@@ -62,7 +62,7 @@ int lan966x_xdp_xmit(struct net_device *dev,
 		struct xdp_frame *xdpf = frames[i];
 		int err;
 
-		err = lan966x_fdma_xmit_xdpf(port, xdpf, NULL, true);
+		err = lan966x_fdma_xmit_xdpf(port, xdpf, NULL, xdpf->len, true);
 		if (err)
 			break;
 
@@ -76,7 +76,6 @@ int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len)
 {
 	struct bpf_prog *xdp_prog = port->xdp_prog;
 	struct lan966x *lan966x = port->lan966x;
-	struct xdp_frame *xdpf;
 	struct xdp_buff xdp;
 	u32 act;
 
@@ -90,11 +89,9 @@ int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len)
 	case XDP_PASS:
 		return FDMA_PASS;
 	case XDP_TX:
-		xdpf = xdp_convert_buff_to_frame(&xdp);
-		if (!xdpf)
-			return FDMA_DROP;
-
-		return lan966x_fdma_xmit_xdpf(port, xdpf, page, false) ?
+		return lan966x_fdma_xmit_xdpf(port, NULL, page,
+					      data_len - IFH_LEN_BYTES,
+					      false) ?
 		       FDMA_DROP : FDMA_TX;
 	case XDP_REDIRECT:
 		if (xdp_do_redirect(port->dev, &xdp, xdp_prog))
-- 
2.38.0

Re: [PATCH net-next] net: lan966x: Don't use xdp_frame when action is XDP_TX

[Alexander Lobakin]

From: Horatiu Vultur <horatiu.vultur@microchip.com>
Date: Thu, 20 Apr 2023 14:11:52 +0200

> When the action of an xdp program was XDP_TX, lan966x was creating
> a xdp_frame and use this one to send the frame back. But it is also
> possible to send back the frame without needing a xdp_frame, because
> it possible to send it back using the page.
> And then once the frame is transmitted is possible to use directly
> page_pool_recycle_direct as lan966x is using page pools.
> This would save some CPU usage on this path.
> 
> Signed-off-by: Horatiu Vultur <horatiu.vultur@microchip.com>

[...]

> @@ -702,6 +704,7 @@ static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use)
>  int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  			   struct xdp_frame *xdpf,
>  			   struct page *page,
> +			   u32 len,
>  			   bool dma_map)

I think you can cut the number of arguments by almost a half:

int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
			   void *ptr, u32 len)
{
	if (len) {
		/* XDP_TX, ptr is page */
		page = ptr;

		dma_sync_here(page, len);
	} else {
		/* XDP_REDIR, ptr is xdp_frame */
		xdpf = ptr;

		dma_map_here(xdpf->data, xdpf->len);
	}

@page and @xdpf are mutually exclusive. When @xdpf is non-null, @len is
excessive (xdpf->len is here), so you can use @len as logical
`len * !dma_map`, i.e. zero for REDIR and the actual frame length for TX.

I generally enjoy seeing how you constantly improve stuff in your driver :)

>  {
>  	struct lan966x *lan966x = port->lan966x;
> @@ -722,6 +725,15 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  		goto out;
>  	}
[...]

Thanks,
Olek

Re: [PATCH net-next] net: lan966x: Don't use xdp_frame when action is XDP_TX

[Horatiu Vultur]

The 04/20/2023 16:49, Alexander Lobakin wrote:
> 
> From: Horatiu Vultur <horatiu.vultur@microchip.com>
> Date: Thu, 20 Apr 2023 14:11:52 +0200

Hi Olek,

> 
> > When the action of an xdp program was XDP_TX, lan966x was creating
> > a xdp_frame and use this one to send the frame back. But it is also
> > possible to send back the frame without needing a xdp_frame, because
> > it possible to send it back using the page.
> > And then once the frame is transmitted is possible to use directly
> > page_pool_recycle_direct as lan966x is using page pools.
> > This would save some CPU usage on this path.
> >
> > Signed-off-by: Horatiu Vultur <horatiu.vultur@microchip.com>
> 
> [...]
> 
> > @@ -702,6 +704,7 @@ static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use)
> >  int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
> >                          struct xdp_frame *xdpf,
> >                          struct page *page,
> > +                        u32 len,
> >                          bool dma_map)
> 
> I think you can cut the number of arguments by almost a half:
> 
> int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>                            void *ptr, u32 len)
> {
>         if (len) {
>                 /* XDP_TX, ptr is page */
>                 page = ptr;
> 
>                 dma_sync_here(page, len);
>         } else {
>                 /* XDP_REDIR, ptr is xdp_frame */
>                 xdpf = ptr;
> 
>                 dma_map_here(xdpf->data, xdpf->len);
>         }
> 
> @page and @xdpf are mutually exclusive. When @xdpf is non-null, @len is
> excessive (xdpf->len is here), so you can use @len as logical
> `len * !dma_map`, i.e. zero for REDIR and the actual frame length for TX.

Thanks for the review. You are right, I can reduce number of arguments,
the reason why I have done it like this, I thought it is a little bit more
clear this way. But I will update as you propose in the next version

> 
> I generally enjoy seeing how you constantly improve stuff in your driver :)
> 
> >  {
> >       struct lan966x *lan966x = port->lan966x;
> > @@ -722,6 +725,15 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
> >               goto out;
> >       }
> [...]
> 
> Thanks,
> Olek

-- 
/Horatiu

Re: [PATCH net-next] net: lan966x: Don't use xdp_frame when action is XDP_TX

[Maciej Fijalkowski]

On Thu, Apr 20, 2023 at 02:11:52PM +0200, Horatiu Vultur wrote:

'net: ' in patch subject is excessive to me

> When the action of an xdp program was XDP_TX, lan966x was creating
> a xdp_frame and use this one to send the frame back. But it is also
> possible to send back the frame without needing a xdp_frame, because
> it possible to send it back using the page.

s/it/it is

> And then once the frame is transmitted is possible to use directly
> page_pool_recycle_direct as lan966x is using page pools.
> This would save some CPU usage on this path.

i remember this optimization gave me noticeable perf improvement, would
you mind sharing it in % on your side?

> 
> Signed-off-by: Horatiu Vultur <horatiu.vultur@microchip.com>
> ---
>  .../ethernet/microchip/lan966x/lan966x_fdma.c | 35 +++++++++++--------
>  .../ethernet/microchip/lan966x/lan966x_main.h |  2 ++
>  .../ethernet/microchip/lan966x/lan966x_xdp.c  | 11 +++---
>  3 files changed, 27 insertions(+), 21 deletions(-)
> 
> diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
> index 2ed76bb61a731..7947259e67e4e 100644
> --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
> +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
> @@ -390,6 +390,7 @@ static void lan966x_fdma_stop_netdev(struct lan966x *lan966x)
>  static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
>  {
>  	struct lan966x_tx *tx = &lan966x->tx;
> +	struct lan966x_rx *rx = &lan966x->rx;
>  	struct lan966x_tx_dcb_buf *dcb_buf;
>  	struct xdp_frame_bulk bq;
>  	struct lan966x_db *db;
> @@ -432,7 +433,8 @@ static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
>  			if (dcb_buf->xdp_ndo)
>  				xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq);
>  			else
> -				xdp_return_frame_rx_napi(dcb_buf->data.xdpf);
> +				page_pool_recycle_direct(rx->page_pool,
> +							 dcb_buf->data.page);
>  		}
>  
>  		clear = true;
> @@ -702,6 +704,7 @@ static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use)
>  int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  			   struct xdp_frame *xdpf,
>  			   struct page *page,
> +			   u32 len,

agreed with Olek regarding arguments reduction here

>  			   bool dma_map)
>  {
>  	struct lan966x *lan966x = port->lan966x;
> @@ -722,6 +725,15 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  		goto out;
>  	}
>  
> +	/* Fill up the buffer */
> +	next_dcb_buf = &tx->dcbs_buf[next_to_use];
> +	next_dcb_buf->use_skb = false;
> +	next_dcb_buf->xdp_ndo = dma_map;

a bit misleading that xdp_ndo is a bool :P

> +	next_dcb_buf->len = len + IFH_LEN_BYTES;
> +	next_dcb_buf->used = true;
> +	next_dcb_buf->ptp = false;
> +	next_dcb_buf->dev = port->dev;
> +
>  	/* Generate new IFH */
>  	if (dma_map) {
>  		if (xdpf->headroom < IFH_LEN_BYTES) {
> @@ -736,16 +748,18 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  
>  		dma_addr = dma_map_single(lan966x->dev,
>  					  xdpf->data - IFH_LEN_BYTES,
> -					  xdpf->len + IFH_LEN_BYTES,
> +					  len + IFH_LEN_BYTES,
>  					  DMA_TO_DEVICE);
>  		if (dma_mapping_error(lan966x->dev, dma_addr)) {
>  			ret = NETDEV_TX_OK;
>  			goto out;
>  		}
>  
> +		next_dcb_buf->data.xdpf = xdpf;
> +
>  		/* Setup next dcb */
>  		lan966x_fdma_tx_setup_dcb(tx, next_to_use,
> -					  xdpf->len + IFH_LEN_BYTES,
> +					  len + IFH_LEN_BYTES,
>  					  dma_addr);
>  	} else {
>  		ifh = page_address(page) + XDP_PACKET_HEADROOM;
> @@ -756,25 +770,18 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  		dma_addr = page_pool_get_dma_addr(page);
>  		dma_sync_single_for_device(lan966x->dev,
>  					   dma_addr + XDP_PACKET_HEADROOM,
> -					   xdpf->len + IFH_LEN_BYTES,
> +					   len + IFH_LEN_BYTES,
>  					   DMA_TO_DEVICE);
>  
> +		next_dcb_buf->data.page = page;
> +
>  		/* Setup next dcb */
>  		lan966x_fdma_tx_setup_dcb(tx, next_to_use,
> -					  xdpf->len + IFH_LEN_BYTES,
> +					  len + IFH_LEN_BYTES,
>  					  dma_addr + XDP_PACKET_HEADROOM);
>  	}
>  
> -	/* Fill up the buffer */
> -	next_dcb_buf = &tx->dcbs_buf[next_to_use];
> -	next_dcb_buf->use_skb = false;
> -	next_dcb_buf->data.xdpf = xdpf;
> -	next_dcb_buf->xdp_ndo = dma_map;
> -	next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES;
>  	next_dcb_buf->dma_addr = dma_addr;
> -	next_dcb_buf->used = true;
> -	next_dcb_buf->ptp = false;
> -	next_dcb_buf->dev = port->dev;
>  
>  	/* Start the transmission */
>  	lan966x_fdma_tx_start(tx, next_to_use);
> diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
> index 851afb0166b19..59da35a2c93d4 100644
> --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
> +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
> @@ -243,6 +243,7 @@ struct lan966x_tx_dcb_buf {
>  	union {
>  		struct sk_buff *skb;
>  		struct xdp_frame *xdpf;
> +		struct page *page;
>  	} data;
>  	u32 len;
>  	u32 used : 1;
> @@ -544,6 +545,7 @@ int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev);
>  int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
>  			   struct xdp_frame *frame,
>  			   struct page *page,
> +			   u32 len,
>  			   bool dma_map);
>  int lan966x_fdma_change_mtu(struct lan966x *lan966x);
>  void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev);
> diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
> index 2e6f486ec67d7..a8ad1f4e431cb 100644
> --- a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
> +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c
> @@ -62,7 +62,7 @@ int lan966x_xdp_xmit(struct net_device *dev,
>  		struct xdp_frame *xdpf = frames[i];
>  		int err;
>  
> -		err = lan966x_fdma_xmit_xdpf(port, xdpf, NULL, true);
> +		err = lan966x_fdma_xmit_xdpf(port, xdpf, NULL, xdpf->len, true);
>  		if (err)
>  			break;
>  
> @@ -76,7 +76,6 @@ int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len)
>  {
>  	struct bpf_prog *xdp_prog = port->xdp_prog;
>  	struct lan966x *lan966x = port->lan966x;
> -	struct xdp_frame *xdpf;
>  	struct xdp_buff xdp;
>  	u32 act;
>  
> @@ -90,11 +89,9 @@ int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len)
>  	case XDP_PASS:
>  		return FDMA_PASS;
>  	case XDP_TX:
> -		xdpf = xdp_convert_buff_to_frame(&xdp);
> -		if (!xdpf)
> -			return FDMA_DROP;
> -
> -		return lan966x_fdma_xmit_xdpf(port, xdpf, page, false) ?
> +		return lan966x_fdma_xmit_xdpf(port, NULL, page,
> +					      data_len - IFH_LEN_BYTES,
> +					      false) ?
>  		       FDMA_DROP : FDMA_TX;
>  	case XDP_REDIRECT:
>  		if (xdp_do_redirect(port->dev, &xdp, xdp_prog))
> -- 
> 2.38.0
> 

Re: [PATCH net-next] net: lan966x: Don't use xdp_frame when action is XDP_TX

[Horatiu Vultur]

The 04/20/2023 22:52, Maciej Fijalkowski wrote:
> [Some people who received this message don't often get email from maciej.fijalkowski@intel.com. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
> 

Hi Maciej,

> 
> On Thu, Apr 20, 2023 at 02:11:52PM +0200, Horatiu Vultur wrote:
> 
> 'net: ' in patch subject is excessive to me

I usually have set this in the subject. I can remove this in the next
version and I will try to keep in mind for other patches for lan966x.

> 
> > When the action of an xdp program was XDP_TX, lan966x was creating
> > a xdp_frame and use this one to send the frame back. But it is also
> > possible to send back the frame without needing a xdp_frame, because
> > it possible to send it back using the page.
> 
> s/it/it is
> 
> > And then once the frame is transmitted is possible to use directly
> > page_pool_recycle_direct as lan966x is using page pools.
> > This would save some CPU usage on this path.
> 
> i remember this optimization gave me noticeable perf improvement, would
> you mind sharing it in % on your side?

The way I have done the measurements, is to measure actually how much
more traffic can be send back. I tried with different frame sizes,
frame size     improvement
64                ~8%
256              ~11%
512               ~8%
1000              ~0%
1500              ~0%

I will make sure do add this to the comments in the next version.

> 
> >
> > Signed-off-by: Horatiu Vultur <horatiu.vultur@microchip.com>
> > ---
> >  .../ethernet/microchip/lan966x/lan966x_fdma.c | 35 +++++++++++--------
> >  .../ethernet/microchip/lan966x/lan966x_main.h |  2 ++
> >  .../ethernet/microchip/lan966x/lan966x_xdp.c  | 11 +++---
> >  3 files changed, 27 insertions(+), 21 deletions(-)
> >
> > diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
> > index 2ed76bb61a731..7947259e67e4e 100644
> > --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
> > +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
> > @@ -390,6 +390,7 @@ static void lan966x_fdma_stop_netdev(struct lan966x *lan966x)
> >  static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
> >  {
> >       struct lan966x_tx *tx = &lan966x->tx;
> > +     struct lan966x_rx *rx = &lan966x->rx;
> >       struct lan966x_tx_dcb_buf *dcb_buf;
> >       struct xdp_frame_bulk bq;
> >       struct lan966x_db *db;
> > @@ -432,7 +433,8 @@ static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight)
> >                       if (dcb_buf->xdp_ndo)
> >                               xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq);
> >                       else
> > -                             xdp_return_frame_rx_napi(dcb_buf->data.xdpf);
> > +                             page_pool_recycle_direct(rx->page_pool,
> > +                                                      dcb_buf->data.page);
> >               }
> >
> >               clear = true;
> > @@ -702,6 +704,7 @@ static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use)
> >  int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
> >                          struct xdp_frame *xdpf,
> >                          struct page *page,
> > +                        u32 len,
> 
> agreed with Olek regarding arguments reduction here

Yes, I will change this in the next version.

> 
> >                          bool dma_map)
> >  {
> >       struct lan966x *lan966x = port->lan966x;
> > @@ -722,6 +725,15 @@ int lan966x_fdma_xmit_xdpf(struct lan966x_port *port,
> >               goto out;
> >       }
> >
> > +     /* Fill up the buffer */
> > +     next_dcb_buf = &tx->dcbs_buf[next_to_use];
> > +     next_dcb_buf->use_skb = false;
> > +     next_dcb_buf->xdp_ndo = dma_map;
> 
> a bit misleading that xdp_ndo is a bool :P

There are few other variables that are misleading :), I need to get to
this and clean it a little bit.

> 
> > +     next_dcb_buf->len = len + IFH_LEN_BYTES;
> > +     next_dcb_buf->used = true;
> > +     next_dcb_buf->ptp = false;
> > +     next_dcb_buf->dev = port->dev;
> > +
> >       /* Generate new IFH */
> >       if (dma_map) {
> >               if (xdpf->headroom < IFH_LEN_BYTES) {
-- 
/Horatiu