From: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
To: bpf@vger.kernel.org, ast@kernel.org, daniel@iogearbox.net,
andrii@kernel.org
Cc: netdev@vger.kernel.org, magnus.karlsson@intel.com,
bjorn@kernel.org, tirthendu.sarkar@intel.com,
maciej.fijalkowski@intel.com, simon.horman@corigine.com,
toke@kernel.org
Subject: [PATCH v4 bpf-next 10/22] xsk: support mbuf on ZC RX
Date: Thu, 15 Jun 2023 19:25:54 +0200 [thread overview]
Message-ID: <20230615172606.349557-11-maciej.fijalkowski@intel.com> (raw)
In-Reply-To: <20230615172606.349557-1-maciej.fijalkowski@intel.com>
Given that skb_shared_info relies on skb_frag_t, in order to support
xskb chaining, introduce xdp_buff_xsk::xskb_list_node and
xsk_buff_pool::xskb_list.
This is needed so ZC drivers can add frags as xskb nodes which will make
it possible to handle it both when producing AF_XDP Rx descriptors as
well as freeing/recycling all the frags that a single frame carries.
Speaking of latter, update xsk_buff_free() to take care of list nodes.
For the former (adding as frags), introduce xsk_buff_add_frag() for ZC
drivers usage that is going to be used to add a frag to to xskb list
from pool.
xsk_buff_get_frag() will be utilized by XDP_TX and, on contrary, will
return xdp_buff.
One of the previous patches added a wrapper for ZC Rx so implement xskb
list walk and production of Rx descriptors there.
Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
---
include/net/xdp_sock_drv.h | 44 +++++++++++++++++++++++++++++++++++++
include/net/xsk_buff_pool.h | 2 ++
net/xdp/xsk.c | 26 +++++++++++++++++++++-
net/xdp/xsk_buff_pool.c | 2 ++
4 files changed, 73 insertions(+), 1 deletion(-)
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index a3e18b958d93..8416ea7ebb3e 100644
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -108,10 +108,45 @@ static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
static inline void xsk_buff_free(struct xdp_buff *xdp)
{
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
+ struct list_head *xskb_list = &xskb->pool->xskb_list;
+ struct xdp_buff_xsk *pos, *tmp;
+ if (likely(!xdp_buff_has_frags(xdp)))
+ goto out;
+
+ list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) {
+ list_del(&pos->xskb_list_node);
+ xp_free(pos);
+ }
+
+ xdp_get_shared_info_from_buff(xdp)->nr_frags = 0;
+out:
xp_free(xskb);
}
+static inline void xsk_buff_add_frag(struct xdp_buff *xdp)
+{
+ struct xdp_buff_xsk *frag = container_of(xdp, struct xdp_buff_xsk, xdp);
+
+ list_add_tail(&frag->xskb_list_node, &frag->pool->xskb_list);
+}
+
+static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first)
+{
+ struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
+ struct xdp_buff *ret = NULL;
+ struct xdp_buff_xsk *frag;
+
+ frag = list_first_entry_or_null(&xskb->pool->xskb_list,
+ struct xdp_buff_xsk, xskb_list_node);
+ if (frag) {
+ list_del(&frag->xskb_list_node);
+ ret = &frag->xdp;
+ }
+
+ return ret;
+}
+
static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
{
xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
@@ -265,6 +300,15 @@ static inline void xsk_buff_free(struct xdp_buff *xdp)
{
}
+static inline void xsk_buff_add_frag(struct xdp_buff *xdp)
+{
+}
+
+static inline struct xdp_buff *xsk_buff_get_frag(struct xdp_buff *first)
+{
+ return NULL;
+}
+
static inline void xsk_buff_discard(struct xdp_buff *xdp)
{
}
diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
index 4dcca163e076..b0bdff26fc88 100644
--- a/include/net/xsk_buff_pool.h
+++ b/include/net/xsk_buff_pool.h
@@ -29,6 +29,7 @@ struct xdp_buff_xsk {
struct xsk_buff_pool *pool;
u64 orig_addr;
struct list_head free_list_node;
+ struct list_head xskb_list_node;
};
#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))
@@ -54,6 +55,7 @@ struct xsk_buff_pool {
struct xdp_umem *umem;
struct work_struct work;
struct list_head free_list;
+ struct list_head xskb_list;
u32 heads_cnt;
u16 queue_id;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 1f20618df5dd..3052d0d3bc7b 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -155,8 +155,32 @@ static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len,
static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
+ u32 frags = xdp_buff_has_frags(xdp);
+ struct xdp_buff_xsk *pos, *tmp;
+ struct list_head *xskb_list;
+ u32 contd = 0;
+ int err;
+
+ if (frags)
+ contd = XDP_PKT_CONTD;
- return __xsk_rcv_zc(xs, xskb, len, 0);
+ err = __xsk_rcv_zc(xs, xskb, len, contd);
+ if (err || likely(!frags))
+ goto out;
+
+ xskb_list = &xskb->pool->xskb_list;
+ list_for_each_entry_safe(pos, tmp, xskb_list, xskb_list_node) {
+ if (list_is_singular(xskb_list))
+ contd = 0;
+ len = pos->xdp.data_end - pos->xdp.data;
+ err = __xsk_rcv_zc(xs, pos, len, contd);
+ if (err)
+ return err;
+ list_del(&pos->xskb_list_node);
+ }
+
+out:
+ return err;
}
static void *xsk_copy_xdp_start(struct xdp_buff *from)
diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index 26f6d304451e..6e2ed66fd430 100644
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -86,6 +86,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
pool->umem = umem;
pool->addrs = umem->addrs;
INIT_LIST_HEAD(&pool->free_list);
+ INIT_LIST_HEAD(&pool->xskb_list);
INIT_LIST_HEAD(&pool->xsk_tx_list);
spin_lock_init(&pool->xsk_tx_list_lock);
spin_lock_init(&pool->cq_lock);
@@ -99,6 +100,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
xskb->pool = pool;
xskb->xdp.frame_sz = umem->chunk_size - umem->headroom;
INIT_LIST_HEAD(&xskb->free_list_node);
+ INIT_LIST_HEAD(&xskb->xskb_list_node);
if (pool->unaligned)
pool->free_heads[i] = xskb;
else
--
2.34.1
next prev parent reply other threads:[~2023-06-15 17:27 UTC|newest]
Thread overview: 41+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-15 17:25 [PATCH v4 bpf-next 00/22] xsk: multi-buffer support Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 01/22] xsk: prepare 'options' in xdp_desc for multi-buffer use Maciej Fijalkowski
2023-06-22 19:02 ` Benjamin Poirier
2023-06-26 13:18 ` Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 02/22] xsk: introduce XSK_USE_SG bind flag for xsk socket Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 03/22] xsk: prepare both copy and zero-copy modes to co-exist Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 04/22] xsk: move xdp_buff's data length check to xsk_rcv_check Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 05/22] xsk: add support for AF_XDP multi-buffer on Rx path Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 06/22] xsk: introduce wrappers and helpers for supporting multi-buffer in Tx path Maciej Fijalkowski
2023-06-20 17:25 ` Toke Høiland-Jørgensen
2023-06-21 8:15 ` Sarkar, Tirthendu
2023-06-21 13:27 ` Toke Høiland-Jørgensen
2023-06-15 17:25 ` [PATCH v4 bpf-next 07/22] xsk: allow core/drivers to test EOP bit Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 08/22] xsk: add support for AF_XDP multi-buffer on Tx path Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 09/22] xsk: discard zero length descriptors in " Maciej Fijalkowski
2023-06-15 17:25 ` Maciej Fijalkowski [this message]
2023-06-15 17:25 ` [PATCH v4 bpf-next 11/22] ice: xsk: add RX multi-buffer support Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 12/22] xsk: support ZC Tx multi-buffer in batch API Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 13/22] xsk: report zero-copy multi-buffer capability via xdp_features Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 14/22] ice: xsk: Tx multi-buffer support Maciej Fijalkowski
2023-06-15 17:25 ` [PATCH v4 bpf-next 15/22] xsk: add multi-buffer documentation Maciej Fijalkowski
2023-06-20 17:34 ` Toke Høiland-Jørgensen
2023-06-21 8:06 ` Magnus Karlsson
2023-06-21 13:30 ` Toke Høiland-Jørgensen
2023-06-21 14:15 ` Magnus Karlsson
2023-06-21 20:34 ` Jakub Kicinski
2023-06-22 8:24 ` Magnus Karlsson
2023-06-22 10:56 ` Toke Høiland-Jørgensen
[not found] ` <ZJx9WkB/dfB5EFjE@boxer>
2023-06-28 20:28 ` Jakub Kicinski
2023-06-28 21:02 ` Toke Høiland-Jørgensen
2023-06-29 20:28 ` Maciej Fijalkowski
2023-06-29 20:57 ` Toke Høiland-Jørgensen
2023-06-30 18:00 ` Maciej Fijalkowski
2023-07-01 13:51 ` Toke Høiland-Jørgensen
2023-06-15 17:26 ` [PATCH v4 bpf-next 16/22] selftests/xsk: transmit and receive multi-buffer packets Maciej Fijalkowski
2023-06-15 17:26 ` [PATCH v4 bpf-next 17/22] selftests/xsk: add basic multi-buffer test Maciej Fijalkowski
2023-06-15 17:26 ` [PATCH v4 bpf-next 18/22] selftests/xsk: add unaligned mode test for multi-buffer Maciej Fijalkowski
2023-06-15 17:26 ` [PATCH v4 bpf-next 19/22] selftests/xsk: add invalid descriptor " Maciej Fijalkowski
2023-06-15 17:26 ` [PATCH v4 bpf-next 20/22] selftests/xsk: add metadata copy test for multi-buff Maciej Fijalkowski
2023-06-15 17:26 ` [PATCH v4 bpf-next 21/22] selftests/xsk: add test for too many frags Maciej Fijalkowski
2023-06-15 17:26 ` [PATCH v4 bpf-next 22/22] selftests/xsk: reset NIC settings to default after running test suite Maciej Fijalkowski
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230615172606.349557-11-maciej.fijalkowski@intel.com \
--to=maciej.fijalkowski@intel.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bjorn@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=magnus.karlsson@intel.com \
--cc=netdev@vger.kernel.org \
--cc=simon.horman@corigine.com \
--cc=tirthendu.sarkar@intel.com \
--cc=toke@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).