* [PATCH bpf-next 01/14] xsk: move xskmap.c to net/xdp/
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 02/14] xsk: move driver interface to xdp_sock_drv.h Björn Töpel
` (12 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
The XSKMAP is partly implemented by net/xdp/xsk.c. Move xskmap.c from
kernel/bpf/ to net/xdp/, which is the logical place for AF_XDP related
code. Also, move AF_XDP struct definitions, and function declarations
only used by AF_XDP internals into net/xdp/xsk.h.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/net/xdp_sock.h | 20 --------------------
kernel/bpf/Makefile | 3 ---
net/xdp/Makefile | 2 +-
net/xdp/xsk.h | 16 ++++++++++++++++
{kernel/bpf => net/xdp}/xskmap.c | 2 ++
5 files changed, 19 insertions(+), 24 deletions(-)
rename {kernel/bpf => net/xdp}/xskmap.c (99%)
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 67191ccaab85..a26d6c80e43d 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -65,22 +65,12 @@ struct xdp_umem {
struct list_head xsk_tx_list;
};
-/* Nodes are linked in the struct xdp_sock map_list field, and used to
- * track which maps a certain socket reside in.
- */
-
struct xsk_map {
struct bpf_map map;
spinlock_t lock; /* Synchronize map updates */
struct xdp_sock *xsk_map[];
};
-struct xsk_map_node {
- struct list_head node;
- struct xsk_map *map;
- struct xdp_sock **map_entry;
-};
-
struct xdp_sock {
/* struct sock must be the first member of struct xdp_sock */
struct sock sk;
@@ -114,7 +104,6 @@ struct xdp_sock {
struct xdp_buff;
#ifdef CONFIG_XDP_SOCKETS
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
-bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs);
/* Used from netdev driver */
bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt);
bool xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr);
@@ -133,10 +122,6 @@ void xsk_clear_rx_need_wakeup(struct xdp_umem *umem);
void xsk_clear_tx_need_wakeup(struct xdp_umem *umem);
bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem);
-void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs,
- struct xdp_sock **map_entry);
-int xsk_map_inc(struct xsk_map *map);
-void xsk_map_put(struct xsk_map *map);
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
void __xsk_map_flush(void);
@@ -242,11 +227,6 @@ static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
return -ENOTSUPP;
}
-static inline bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
-{
- return false;
-}
-
static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
{
return false;
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index f2d7be596966..c4334132816e 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -12,9 +12,6 @@ obj-$(CONFIG_BPF_JIT) += dispatcher.o
ifeq ($(CONFIG_NET),y)
obj-$(CONFIG_BPF_SYSCALL) += devmap.o
obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
-ifeq ($(CONFIG_XDP_SOCKETS),y)
-obj-$(CONFIG_BPF_SYSCALL) += xskmap.o
-endif
obj-$(CONFIG_BPF_SYSCALL) += offload.o
endif
ifeq ($(CONFIG_PERF_EVENTS),y)
diff --git a/net/xdp/Makefile b/net/xdp/Makefile
index 71e2bdafb2ce..90b5460d6166 100644
--- a/net/xdp/Makefile
+++ b/net/xdp/Makefile
@@ -1,3 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o xsk_queue.o
+obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o xsk_queue.o xskmap.o
obj-$(CONFIG_XDP_SOCKETS_DIAG) += xsk_diag.o
diff --git a/net/xdp/xsk.h b/net/xdp/xsk.h
index 4cfd106bdb53..d6a0979050e6 100644
--- a/net/xdp/xsk.h
+++ b/net/xdp/xsk.h
@@ -17,9 +17,25 @@ struct xdp_mmap_offsets_v1 {
struct xdp_ring_offset_v1 cr;
};
+/* Nodes are linked in the struct xdp_sock map_list field, and used to
+ * track which maps a certain socket reside in.
+ */
+
+struct xsk_map_node {
+ struct list_head node;
+ struct xsk_map *map;
+ struct xdp_sock **map_entry;
+};
+
static inline struct xdp_sock *xdp_sk(struct sock *sk)
{
return (struct xdp_sock *)sk;
}
+bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs);
+void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs,
+ struct xdp_sock **map_entry);
+int xsk_map_inc(struct xsk_map *map);
+void xsk_map_put(struct xsk_map *map);
+
#endif /* XSK_H_ */
diff --git a/kernel/bpf/xskmap.c b/net/xdp/xskmap.c
similarity index 99%
rename from kernel/bpf/xskmap.c
rename to net/xdp/xskmap.c
index 2cc5c8f4c800..1dc7208c71ba 100644
--- a/kernel/bpf/xskmap.c
+++ b/net/xdp/xskmap.c
@@ -9,6 +9,8 @@
#include <linux/slab.h>
#include <linux/sched.h>
+#include "xsk.h"
+
int xsk_map_inc(struct xsk_map *map)
{
bpf_map_inc(&map->map);
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 02/14] xsk: move driver interface to xdp_sock_drv.h
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 01/14] xsk: move xskmap.c to net/xdp/ Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 03/14] xsk: move defines only used by AF_XDP internals to xsk.h Björn Töpel
` (11 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: maximmi, maciej.fijalkowski, bjorn.topel
From: Magnus Karlsson <magnus.karlsson@intel.com>
Move the AF_XDP zero-copy driver interface to its own include file
called xdp_sock_drv.h. This, hopefully, will make it more clear for
NIC driver implementors to know what functions to use for zero-copy
support.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
---
drivers/net/ethernet/intel/i40e/i40e_main.c | 2 +-
drivers/net/ethernet/intel/i40e/i40e_xsk.c | 2 +-
drivers/net/ethernet/intel/ice/ice_xsk.c | 2 +-
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 2 +-
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 2 +-
.../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 2 +-
.../ethernet/mellanox/mlx5/core/en/xsk/tx.h | 2 +-
.../ethernet/mellanox/mlx5/core/en/xsk/umem.c | 2 +-
include/net/xdp_sock.h | 203 +----------------
include/net/xdp_sock_drv.h | 207 ++++++++++++++++++
net/ethtool/channels.c | 2 +-
net/ethtool/ioctl.c | 2 +-
net/xdp/xdp_umem.h | 2 +-
net/xdp/xsk.c | 2 +-
14 files changed, 227 insertions(+), 207 deletions(-)
create mode 100644 include/net/xdp_sock_drv.h
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 2a037ec244b9..d6b2db4f2c65 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -11,7 +11,7 @@
#include "i40e_diag.h"
#include "i40e_xsk.h"
#include <net/udp_tunnel.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
/* All i40e tracepoints are defined by the include below, which
* must be included exactly once across the whole kernel with
* CREATE_TRACE_POINTS defined
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index 0b7d29192b2c..452bba7bc4ff 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -2,7 +2,7 @@
/* Copyright(c) 2018 Intel Corporation. */
#include <linux/bpf_trace.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "i40e.h"
diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c
index 8279db15e870..955b0fbb7c9a 100644
--- a/drivers/net/ethernet/intel/ice/ice_xsk.c
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.c
@@ -2,7 +2,7 @@
/* Copyright (c) 2019, Intel Corporation. */
#include <linux/bpf_trace.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "ice.h"
#include "ice_base.h"
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
index 74b540ebb3dc..5b6edbd8a4ed 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
@@ -2,7 +2,7 @@
/* Copyright(c) 2018 Intel Corporation. */
#include <linux/bpf_trace.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "ixgbe.h"
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index c4a7fb4ecd14..b04b99396f65 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -31,7 +31,7 @@
*/
#include <linux/bpf_trace.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include "en/xdp.h"
#include "en/params.h"
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
index cab0e93497ae..a8e11adbf426 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
@@ -5,7 +5,7 @@
#define __MLX5_EN_XSK_RX_H__
#include "en.h"
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
/* RX data path */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
index 79b487d89757..39fa0a705856 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
@@ -5,7 +5,7 @@
#define __MLX5_EN_XSK_TX_H__
#include "en.h"
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
/* TX data path */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
index 4baaa5788320..5e49fdb564b3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Mellanox Technologies. */
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include "umem.h"
#include "setup.h"
#include "en/params.h"
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index a26d6c80e43d..6a986dcbc336 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -15,6 +15,7 @@
struct net_device;
struct xsk_queue;
+struct xdp_buff;
/* Masks for xdp_umem_page flags.
* The low 12-bits of the addr will be 0 since this is the page address, so we
@@ -101,27 +102,9 @@ struct xdp_sock {
spinlock_t map_list_lock;
};
-struct xdp_buff;
#ifdef CONFIG_XDP_SOCKETS
-int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
-/* Used from netdev driver */
-bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt);
-bool xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr);
-void xsk_umem_release_addr(struct xdp_umem *umem);
-void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries);
-bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc);
-void xsk_umem_consume_tx_done(struct xdp_umem *umem);
-struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries);
-struct xdp_umem_fq_reuse *xsk_reuseq_swap(struct xdp_umem *umem,
- struct xdp_umem_fq_reuse *newq);
-void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq);
-struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, u16 queue_id);
-void xsk_set_rx_need_wakeup(struct xdp_umem *umem);
-void xsk_set_tx_need_wakeup(struct xdp_umem *umem);
-void xsk_clear_rx_need_wakeup(struct xdp_umem *umem);
-void xsk_clear_tx_need_wakeup(struct xdp_umem *umem);
-bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem);
+int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
void __xsk_map_flush(void);
@@ -153,125 +136,24 @@ static inline u64 xsk_umem_add_offset_to_addr(u64 addr)
return xsk_umem_extract_addr(addr) + xsk_umem_extract_offset(addr);
}
-static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)
-{
- unsigned long page_addr;
-
- addr = xsk_umem_add_offset_to_addr(addr);
- page_addr = (unsigned long)umem->pages[addr >> PAGE_SHIFT].addr;
-
- return (char *)(page_addr & PAGE_MASK) + (addr & ~PAGE_MASK);
-}
-
-static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr)
-{
- addr = xsk_umem_add_offset_to_addr(addr);
-
- return umem->pages[addr >> PAGE_SHIFT].dma + (addr & ~PAGE_MASK);
-}
-
-/* Reuse-queue aware version of FILL queue helpers */
-static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- if (rq->length >= cnt)
- return true;
-
- return xsk_umem_has_addrs(umem, cnt - rq->length);
-}
-
-static inline bool xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- if (!rq->length)
- return xsk_umem_peek_addr(umem, addr);
-
- *addr = rq->handles[rq->length - 1];
- return addr;
-}
-
-static inline void xsk_umem_release_addr_rq(struct xdp_umem *umem)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- if (!rq->length)
- xsk_umem_release_addr(umem);
- else
- rq->length--;
-}
-
-static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- rq->handles[rq->length++] = addr;
-}
-
-/* Handle the offset appropriately depending on aligned or unaligned mode.
- * For unaligned mode, we store the offset in the upper 16-bits of the address.
- * For aligned mode, we simply add the offset to the address.
- */
-static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 address,
- u64 offset)
-{
- if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG)
- return address + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
- else
- return address + offset;
-}
#else
+
static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
return -ENOTSUPP;
}
-static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
-{
- return false;
-}
-
-static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
-{
- return NULL;
-}
-
-static inline void xsk_umem_release_addr(struct xdp_umem *umem)
-{
-}
-
-static inline void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
-{
-}
-
-static inline bool xsk_umem_consume_tx(struct xdp_umem *umem,
- struct xdp_desc *desc)
-{
- return false;
-}
-
-static inline void xsk_umem_consume_tx_done(struct xdp_umem *umem)
-{
-}
-
-static inline struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries)
+static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
{
- return NULL;
+ return -EOPNOTSUPP;
}
-static inline struct xdp_umem_fq_reuse *xsk_reuseq_swap(
- struct xdp_umem *umem,
- struct xdp_umem_fq_reuse *newq)
-{
- return NULL;
-}
-static inline void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq)
+static inline void __xsk_map_flush(void)
{
}
-static inline struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev,
- u16 queue_id)
+static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
+ u32 key)
{
return NULL;
}
@@ -291,75 +173,6 @@ static inline u64 xsk_umem_add_offset_to_addr(u64 addr)
return 0;
}
-static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)
-{
- return NULL;
-}
-
-static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr)
-{
- return 0;
-}
-
-static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
-{
- return false;
-}
-
-static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
-{
- return NULL;
-}
-
-static inline void xsk_umem_release_addr_rq(struct xdp_umem *umem)
-{
-}
-
-static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr)
-{
-}
-
-static inline void xsk_set_rx_need_wakeup(struct xdp_umem *umem)
-{
-}
-
-static inline void xsk_set_tx_need_wakeup(struct xdp_umem *umem)
-{
-}
-
-static inline void xsk_clear_rx_need_wakeup(struct xdp_umem *umem)
-{
-}
-
-static inline void xsk_clear_tx_need_wakeup(struct xdp_umem *umem)
-{
-}
-
-static inline bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
-{
- return false;
-}
-
-static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 handle,
- u64 offset)
-{
- return 0;
-}
-
-static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
-{
- return -EOPNOTSUPP;
-}
-
-static inline void __xsk_map_flush(void)
-{
-}
-
-static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
- u32 key)
-{
- return NULL;
-}
#endif /* CONFIG_XDP_SOCKETS */
#endif /* _LINUX_XDP_SOCK_H */
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
new file mode 100644
index 000000000000..98dd6962e6d4
--- /dev/null
+++ b/include/net/xdp_sock_drv.h
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Interface for implementing AF_XDP zero-copy support in drivers.
+ * Copyright(c) 2020 Intel Corporation.
+ */
+
+#ifndef _LINUX_XDP_SOCK_DRV_H
+#define _LINUX_XDP_SOCK_DRV_H
+
+#include <net/xdp_sock.h>
+
+#ifdef CONFIG_XDP_SOCKETS
+
+bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt);
+bool xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr);
+void xsk_umem_release_addr(struct xdp_umem *umem);
+void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries);
+bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc);
+void xsk_umem_consume_tx_done(struct xdp_umem *umem);
+struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries);
+struct xdp_umem_fq_reuse *xsk_reuseq_swap(struct xdp_umem *umem,
+ struct xdp_umem_fq_reuse *newq);
+void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq);
+struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, u16 queue_id);
+void xsk_set_rx_need_wakeup(struct xdp_umem *umem);
+void xsk_set_tx_need_wakeup(struct xdp_umem *umem);
+void xsk_clear_rx_need_wakeup(struct xdp_umem *umem);
+void xsk_clear_tx_need_wakeup(struct xdp_umem *umem);
+bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem);
+
+static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)
+{
+ unsigned long page_addr;
+
+ addr = xsk_umem_add_offset_to_addr(addr);
+ page_addr = (unsigned long)umem->pages[addr >> PAGE_SHIFT].addr;
+
+ return (char *)(page_addr & PAGE_MASK) + (addr & ~PAGE_MASK);
+}
+
+static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr)
+{
+ addr = xsk_umem_add_offset_to_addr(addr);
+
+ return umem->pages[addr >> PAGE_SHIFT].dma + (addr & ~PAGE_MASK);
+}
+
+/* Reuse-queue aware version of FILL queue helpers */
+static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
+{
+ struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
+
+ if (rq->length >= cnt)
+ return true;
+
+ return xsk_umem_has_addrs(umem, cnt - rq->length);
+}
+
+static inline bool xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
+{
+ struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
+
+ if (!rq->length)
+ return xsk_umem_peek_addr(umem, addr);
+
+ *addr = rq->handles[rq->length - 1];
+ return addr;
+}
+
+static inline void xsk_umem_release_addr_rq(struct xdp_umem *umem)
+{
+ struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
+
+ if (!rq->length)
+ xsk_umem_release_addr(umem);
+ else
+ rq->length--;
+}
+
+static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr)
+{
+ struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
+
+ rq->handles[rq->length++] = addr;
+}
+
+/* Handle the offset appropriately depending on aligned or unaligned mode.
+ * For unaligned mode, we store the offset in the upper 16-bits of the address.
+ * For aligned mode, we simply add the offset to the address.
+ */
+static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 address,
+ u64 offset)
+{
+ if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG)
+ return address + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
+ else
+ return address + offset;
+}
+
+#else
+
+static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
+{
+ return false;
+}
+
+static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
+{
+ return NULL;
+}
+
+static inline void xsk_umem_release_addr(struct xdp_umem *umem)
+{
+}
+
+static inline void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
+{
+}
+
+static inline bool xsk_umem_consume_tx(struct xdp_umem *umem,
+ struct xdp_desc *desc)
+{
+ return false;
+}
+
+static inline void xsk_umem_consume_tx_done(struct xdp_umem *umem)
+{
+}
+
+static inline struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries)
+{
+ return NULL;
+}
+
+static inline struct xdp_umem_fq_reuse *xsk_reuseq_swap(
+ struct xdp_umem *umem, struct xdp_umem_fq_reuse *newq)
+{
+ return NULL;
+}
+
+static inline void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq)
+{
+}
+
+static inline struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev,
+ u16 queue_id)
+{
+ return NULL;
+}
+
+static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)
+{
+ return NULL;
+}
+
+static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr)
+{
+ return 0;
+}
+
+static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
+{
+ return false;
+}
+
+static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
+{
+ return NULL;
+}
+
+static inline void xsk_umem_release_addr_rq(struct xdp_umem *umem)
+{
+}
+
+static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr)
+{
+}
+
+static inline void xsk_set_rx_need_wakeup(struct xdp_umem *umem)
+{
+}
+
+static inline void xsk_set_tx_need_wakeup(struct xdp_umem *umem)
+{
+}
+
+static inline void xsk_clear_rx_need_wakeup(struct xdp_umem *umem)
+{
+}
+
+static inline void xsk_clear_tx_need_wakeup(struct xdp_umem *umem)
+{
+}
+
+static inline bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
+{
+ return false;
+}
+
+static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 handle,
+ u64 offset)
+{
+ return 0;
+}
+
+#endif /* CONFIG_XDP_SOCKETS */
+
+#endif /* _LINUX_XDP_SOCK_DRV_H */
diff --git a/net/ethtool/channels.c b/net/ethtool/channels.c
index 389924b65d05..658a8580b464 100644
--- a/net/ethtool/channels.c
+++ b/net/ethtool/channels.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include "netlink.h"
#include "common.h"
diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c
index 226d5ecdd567..c54eb042fb93 100644
--- a/net/ethtool/ioctl.c
+++ b/net/ethtool/ioctl.c
@@ -24,7 +24,7 @@
#include <linux/sched/signal.h>
#include <linux/net.h>
#include <net/devlink.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include <net/flow_offload.h>
#include <linux/ethtool_netlink.h>
#include <generated/utsrelease.h>
diff --git a/net/xdp/xdp_umem.h b/net/xdp/xdp_umem.h
index a63a9fb251f5..32067fe98f65 100644
--- a/net/xdp/xdp_umem.h
+++ b/net/xdp/xdp_umem.h
@@ -6,7 +6,7 @@
#ifndef XDP_UMEM_H_
#define XDP_UMEM_H_
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
u16 queue_id, u16 flags);
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 45ffd67b367d..8bda654e82ec 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -22,7 +22,7 @@
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/rculist.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "xsk_queue.h"
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 03/14] xsk: move defines only used by AF_XDP internals to xsk.h
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 01/14] xsk: move xskmap.c to net/xdp/ Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 02/14] xsk: move driver interface to xdp_sock_drv.h Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 04/14] xsk: introduce AF_XDP buffer allocation API Björn Töpel
` (10 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
Move the XSK_NEXT_PG_CONTIG_{MASK,SHIFT}, and
XDP_UMEM_USES_NEED_WAKEUP defines from xdp_sock.h to the AF_XDP
internal xsk.h file. Also, start using the BIT{,_ULL} macro instead of
explicit shifts.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/net/xdp_sock.h | 14 --------------
net/xdp/xsk.h | 14 ++++++++++++++
net/xdp/xsk_queue.h | 2 ++
3 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 6a986dcbc336..fb7fe3060175 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -17,13 +17,6 @@ struct net_device;
struct xsk_queue;
struct xdp_buff;
-/* Masks for xdp_umem_page flags.
- * The low 12-bits of the addr will be 0 since this is the page address, so we
- * can use them for flags.
- */
-#define XSK_NEXT_PG_CONTIG_SHIFT 0
-#define XSK_NEXT_PG_CONTIG_MASK (1ULL << XSK_NEXT_PG_CONTIG_SHIFT)
-
struct xdp_umem_page {
void *addr;
dma_addr_t dma;
@@ -35,13 +28,6 @@ struct xdp_umem_fq_reuse {
u64 handles[];
};
-/* Flags for the umem flags field.
- *
- * The NEED_WAKEUP flag is 1 due to the reuse of the flags field for public
- * flags. See inlude/uapi/include/linux/if_xdp.h.
- */
-#define XDP_UMEM_USES_NEED_WAKEUP (1 << 1)
-
struct xdp_umem {
struct xsk_queue *fq;
struct xsk_queue *cq;
diff --git a/net/xdp/xsk.h b/net/xdp/xsk.h
index d6a0979050e6..455ddd480f3d 100644
--- a/net/xdp/xsk.h
+++ b/net/xdp/xsk.h
@@ -4,6 +4,20 @@
#ifndef XSK_H_
#define XSK_H_
+/* Masks for xdp_umem_page flags.
+ * The low 12-bits of the addr will be 0 since this is the page address, so we
+ * can use them for flags.
+ */
+#define XSK_NEXT_PG_CONTIG_SHIFT 0
+#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT)
+
+/* Flags for the umem flags field.
+ *
+ * The NEED_WAKEUP flag is 1 due to the reuse of the flags field for public
+ * flags. See inlude/uapi/include/linux/if_xdp.h.
+ */
+#define XDP_UMEM_USES_NEED_WAKEUP BIT(1)
+
struct xdp_ring_offset_v1 {
__u64 producer;
__u64 consumer;
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index 648733ec24ac..a322a7dac58c 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -10,6 +10,8 @@
#include <linux/if_xdp.h>
#include <net/xdp_sock.h>
+#include "xsk.h"
+
struct xdp_ring {
u32 producer ____cacheline_aligned_in_smp;
u32 consumer ____cacheline_aligned_in_smp;
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 04/14] xsk: introduce AF_XDP buffer allocation API
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (2 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 03/14] xsk: move defines only used by AF_XDP internals to xsk.h Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-08 11:55 ` Maxim Mikityanskiy
2020-05-07 10:42 ` [PATCH bpf-next 05/14] i40e: refactor rx_bi accesses Björn Töpel
` (9 subsequent siblings)
13 siblings, 1 reply; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
In order to simplify AF_XDP zero-copy enablement for NIC driver
developers, a new AF_XDP buffer allocation API is added. The
implementation is based on a single core (single producer/consumer)
buffer pool for the AF_XDP UMEM.
A buffer is allocated using the xsk_buff_alloc() function, and
returned using xsk_buff_free(). If a buffer is disassociated with the
pool, e.g. when a buffer is passed to an AF_XDP socket, a buffer is
said to be released. Currently, the release function is only used by
the AF_XDP internals and not visible to the driver.
Drivers using this API should register the XDP memory model with the
new MEM_TYPE_XSK_BUFF_POOL type.
The API is defined in net/xdp_sock_drv.h.
The buffer type is struct xdp_buff, and follows the lifetime of
regular xdp_buffs, i.e. the lifetime of an xdp_buff is restricted to
a NAPI context. In other words, the API is not replacing xdp_frames.
In addition to introducing the API and implementations, the AF_XDP
core is migrated to use the new APIs.
rfc->v1: Fixed build errors/warnings for m68k and riscv. (kbuild test
robot)
Added headroom/chunk size getter. (Maxim/Björn)
Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/net/xdp.h | 4 +-
include/net/xdp_sock.h | 2 +
include/net/xdp_sock_drv.h | 152 ++++++++++++
include/net/xsk_buff_pool.h | 54 +++++
include/trace/events/xdp.h | 3 +-
net/core/xdp.c | 14 +-
net/xdp/Makefile | 1 +
net/xdp/xdp_umem.c | 19 +-
net/xdp/xsk.c | 147 +++++-------
net/xdp/xsk_buff_pool.c | 462 ++++++++++++++++++++++++++++++++++++
net/xdp/xsk_diag.c | 2 +-
net/xdp/xsk_queue.h | 59 +++--
12 files changed, 800 insertions(+), 119 deletions(-)
create mode 100644 include/net/xsk_buff_pool.h
create mode 100644 net/xdp/xsk_buff_pool.c
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 3cc6d5d84aa4..83173e4d306c 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -38,6 +38,7 @@ enum xdp_mem_type {
MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
MEM_TYPE_PAGE_POOL,
MEM_TYPE_ZERO_COPY,
+ MEM_TYPE_XSK_BUFF_POOL,
MEM_TYPE_MAX,
};
@@ -101,7 +102,8 @@ struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
int metasize;
int headroom;
- if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
+ if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
+ xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
return xdp_convert_zc_to_xdp_frame(xdp);
/* Assure headroom is available for storing info */
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index fb7fe3060175..6e7265f63c04 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -31,11 +31,13 @@ struct xdp_umem_fq_reuse {
struct xdp_umem {
struct xsk_queue *fq;
struct xsk_queue *cq;
+ struct xsk_buff_pool *pool;
struct xdp_umem_page *pages;
u64 chunk_mask;
u64 size;
u32 headroom;
u32 chunk_size_nohr;
+ u32 chunk_size;
struct user_struct *user;
refcount_t users;
struct work_struct work;
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index 98dd6962e6d4..5a0970d4c44c 100644
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -7,6 +7,7 @@
#define _LINUX_XDP_SOCK_DRV_H
#include <net/xdp_sock.h>
+#include <net/xsk_buff_pool.h>
#ifdef CONFIG_XDP_SOCKETS
@@ -96,6 +97,87 @@ static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 address,
return address + offset;
}
+static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
+{
+ return XDP_PACKET_HEADROOM + umem->headroom;
+}
+
+static inline u32 xsk_umem_get_chunk_size(struct xdp_umem *umem)
+{
+ return umem->chunk_size;
+}
+
+static inline u32 xsk_umem_get_rx_frame_size(struct xdp_umem *umem)
+{
+ return xsk_umem_get_chunk_size(umem) - xsk_umem_get_headroom(umem);
+}
+
+static inline void xsk_buff_set_rxq_info(struct xdp_umem *umem,
+ struct xdp_rxq_info *rxq)
+{
+ xp_set_rxq_info(umem->pool, rxq);
+}
+
+static inline void xsk_buff_dma_unmap(struct xdp_umem *umem,
+ unsigned long attrs)
+{
+ xp_dma_unmap(umem->pool, attrs);
+}
+
+static inline int xsk_buff_dma_map(struct xdp_umem *umem, struct device *dev,
+ unsigned long attrs)
+{
+ return xp_dma_map(umem->pool, dev, attrs, umem->pgs, umem->npgs);
+}
+
+static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
+{
+ struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
+
+ return xp_get_dma(xskb);
+}
+
+static inline struct xdp_buff *xsk_buff_alloc(struct xdp_umem *umem)
+{
+ return xp_alloc(umem->pool);
+}
+
+static inline bool xsk_buff_can_alloc(struct xdp_umem *umem, u32 count)
+{
+ return xp_can_alloc(umem->pool, count);
+}
+
+static inline void xsk_buff_free(struct xdp_buff *xdp)
+{
+ struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
+
+ xp_free(xskb);
+}
+
+static inline dma_addr_t xsk_buff_raw_get_dma(struct xdp_umem *umem, u64 addr)
+{
+ return xp_raw_get_dma(umem->pool, addr);
+}
+
+static inline void *xsk_buff_raw_get_data(struct xdp_umem *umem, u64 addr)
+{
+ return xp_raw_get_data(umem->pool, addr);
+}
+
+static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
+{
+ struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
+
+ xp_dma_sync_for_cpu(xskb);
+}
+
+static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
+ dma_addr_t dma,
+ size_t size)
+{
+ xp_dma_sync_for_device(umem->pool, dma, size);
+}
+
#else
static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
@@ -202,6 +284,76 @@ static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 handle,
return 0;
}
+static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
+{
+ return 0;
+}
+
+static inline u32 xsk_umem_get_chunk_size(struct xdp_umem *umem)
+{
+ return 0;
+}
+
+static inline u32 xsk_umem_get_rx_frame_size(struct xdp_umem *umem)
+{
+ return 0;
+}
+
+static inline void xsk_buff_set_rxq_info(struct xdp_umem *umem,
+ struct xdp_rxq_info *rxq)
+{
+}
+
+static inline void xsk_buff_dma_unmap(struct xdp_umem *umem,
+ unsigned long attrs)
+{
+}
+
+static inline int xsk_buff_dma_map(struct xdp_umem *umem, struct device *dev,
+ unsigned long attrs)
+{
+ return 0;
+}
+
+static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
+{
+ return 0;
+}
+
+static inline struct xdp_buff *xsk_buff_alloc(struct xdp_umem *umem)
+{
+ return NULL;
+}
+
+static inline bool xsk_buff_can_alloc(struct xdp_umem *umem, u32 count)
+{
+ return false;
+}
+
+static inline void xsk_buff_free(struct xdp_buff *xdp)
+{
+}
+
+static inline dma_addr_t xsk_buff_raw_get_dma(struct xdp_umem *umem, u64 addr)
+{
+ return 0;
+}
+
+static inline void *xsk_buff_raw_get_data(struct xdp_umem *umem, u64 addr)
+{
+ return NULL;
+}
+
+static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
+{
+}
+
+static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
+ dma_addr_t dma,
+ size_t size)
+{
+}
+
#endif /* CONFIG_XDP_SOCKETS */
#endif /* _LINUX_XDP_SOCK_DRV_H */
diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
new file mode 100644
index 000000000000..9abef166441d
--- /dev/null
+++ b/include/net/xsk_buff_pool.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2020 Intel Corporation. */
+
+#ifndef XSK_BUFF_POOL_H_
+#define XSK_BUFF_POOL_H_
+
+#include <linux/types.h>
+#include <linux/dma-mapping.h>
+#include <net/xdp.h>
+
+struct xsk_buff_pool;
+struct xdp_rxq_info;
+struct xsk_queue;
+struct xdp_desc;
+struct device;
+struct page;
+
+struct xdp_buff_xsk {
+ struct xdp_buff xdp;
+ dma_addr_t dma;
+ struct xsk_buff_pool *pool;
+ bool unaligned;
+ u64 orig_addr;
+ struct list_head free_list_node;
+};
+
+/* AF_XDP core. */
+struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
+ u32 chunk_size, u32 headroom, u64 size,
+ bool unaligned);
+void xp_set_fq(struct xsk_buff_pool *pool, struct xsk_queue *fq);
+void xp_destroy(struct xsk_buff_pool *pool);
+void xp_release(struct xdp_buff_xsk *xskb);
+u64 xp_get_handle(struct xdp_buff_xsk *xskb);
+bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
+
+/* AF_XDP, and XDP core. */
+void xp_free(struct xdp_buff_xsk *xskb);
+
+/* AF_XDP ZC drivers, via xdp_sock_buff.h */
+void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq);
+int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
+ unsigned long attrs, struct page **pages, u32 nr_pages);
+void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs);
+struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
+bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
+void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
+dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
+dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb);
+void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb);
+void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
+ size_t size);
+
+#endif /* XSK_BUFF_POOL_H_ */
diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index b95d65e8c628..48547a12fa27 100644
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -287,7 +287,8 @@ TRACE_EVENT(xdp_devmap_xmit,
FN(PAGE_SHARED) \
FN(PAGE_ORDER0) \
FN(PAGE_POOL) \
- FN(ZERO_COPY)
+ FN(ZERO_COPY) \
+ FN(XSK_BUFF_POOL)
#define __MEM_TYPE_TP_FN(x) \
TRACE_DEFINE_ENUM(MEM_TYPE_##x);
diff --git a/net/core/xdp.c b/net/core/xdp.c
index 4c7ea85486af..89053ef8333b 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -16,6 +16,7 @@
#include <net/xdp.h>
#include <net/xdp_priv.h> /* struct xdp_mem_allocator */
#include <trace/events/xdp.h>
+#include <net/xdp_sock_drv.h>
#define REG_STATE_NEW 0x0
#define REG_STATE_REGISTERED 0x1
@@ -360,7 +361,7 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
* of xdp_frames/pages in those cases.
*/
static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
- unsigned long handle)
+ unsigned long handle, struct xdp_buff *xdp)
{
struct xdp_mem_allocator *xa;
struct page *page;
@@ -389,6 +390,11 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
xa->zc_alloc->free(xa->zc_alloc, handle);
rcu_read_unlock();
+ break;
+ case MEM_TYPE_XSK_BUFF_POOL:
+ /* NB! Only valid from an xdp_buff! */
+ xsk_buff_free(xdp);
+ break;
default:
/* Not possible, checked in xdp_rxq_info_reg_mem_model() */
break;
@@ -397,19 +403,19 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
void xdp_return_frame(struct xdp_frame *xdpf)
{
- __xdp_return(xdpf->data, &xdpf->mem, false, 0);
+ __xdp_return(xdpf->data, &xdpf->mem, false, 0, NULL);
}
EXPORT_SYMBOL_GPL(xdp_return_frame);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
{
- __xdp_return(xdpf->data, &xdpf->mem, true, 0);
+ __xdp_return(xdpf->data, &xdpf->mem, true, 0, NULL);
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
void xdp_return_buff(struct xdp_buff *xdp)
{
- __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle);
+ __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle, xdp);
}
EXPORT_SYMBOL_GPL(xdp_return_buff);
diff --git a/net/xdp/Makefile b/net/xdp/Makefile
index 90b5460d6166..30cdc4315f42 100644
--- a/net/xdp/Makefile
+++ b/net/xdp/Makefile
@@ -1,3 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o xsk_queue.o xskmap.o
+obj-$(CONFIG_XDP_SOCKETS) += xsk_buff_pool.o
obj-$(CONFIG_XDP_SOCKETS_DIAG) += xsk_diag.o
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 37ace3bc0d48..7f04688045d5 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -245,7 +245,7 @@ static void xdp_umem_release(struct xdp_umem *umem)
}
xsk_reuseq_destroy(umem);
-
+ xp_destroy(umem->pool);
xdp_umem_unmap_pages(umem);
xdp_umem_unpin_pages(umem);
@@ -390,6 +390,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
umem->size = size;
umem->headroom = headroom;
umem->chunk_size_nohr = chunk_size - headroom;
+ umem->chunk_size = chunk_size;
umem->npgs = size / PAGE_SIZE;
umem->pgs = NULL;
umem->user = NULL;
@@ -415,11 +416,21 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
}
err = xdp_umem_map_pages(umem);
- if (!err)
- return 0;
+ if (err)
+ goto out_pages;
- kvfree(umem->pages);
+ umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size,
+ headroom, size, unaligned_chunks);
+ if (!umem->pool) {
+ err = -ENOMEM;
+ goto out_unmap;
+ }
+ return 0;
+out_unmap:
+ xdp_umem_unmap_pages(umem);
+out_pages:
+ kvfree(umem->pages);
out_pin:
xdp_umem_unpin_pages(umem);
out_account:
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 8bda654e82ec..6933f0d494ba 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -117,76 +117,67 @@ bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
}
EXPORT_SYMBOL(xsk_umem_uses_need_wakeup);
-/* If a buffer crosses a page boundary, we need to do 2 memcpy's, one for
- * each page. This is only required in copy mode.
- */
-static void __xsk_rcv_memcpy(struct xdp_umem *umem, u64 addr, void *from_buf,
- u32 len, u32 metalen)
+static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
- void *to_buf = xdp_umem_get_data(umem, addr);
-
- addr = xsk_umem_add_offset_to_addr(addr);
- if (xskq_cons_crosses_non_contig_pg(umem, addr, len + metalen)) {
- void *next_pg_addr = umem->pages[(addr >> PAGE_SHIFT) + 1].addr;
- u64 page_start = addr & ~(PAGE_SIZE - 1);
- u64 first_len = PAGE_SIZE - (addr - page_start);
-
- memcpy(to_buf, from_buf, first_len);
- memcpy(next_pg_addr, from_buf + first_len,
- len + metalen - first_len);
+ struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
+ u64 addr;
+ int err;
- return;
+ addr = xp_get_handle(xskb);
+ err = xskq_prod_reserve_desc(xs->rx, addr, len);
+ if (err) {
+ xs->rx_dropped++;
+ return err;
}
- memcpy(to_buf, from_buf, len + metalen);
+ xp_release(xskb);
+ return 0;
}
-static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
+static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len)
{
- u64 offset = xs->umem->headroom;
- u64 addr, memcpy_addr;
- void *from_buf;
+ void *from_buf, *to_buf;
u32 metalen;
- int err;
-
- if (!xskq_cons_peek_addr(xs->umem->fq, &addr, xs->umem) ||
- len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
- xs->rx_dropped++;
- return -ENOSPC;
- }
- if (unlikely(xdp_data_meta_unsupported(xdp))) {
- from_buf = xdp->data;
+ if (unlikely(xdp_data_meta_unsupported(from))) {
+ from_buf = from->data;
+ to_buf = to->data;
metalen = 0;
} else {
- from_buf = xdp->data_meta;
- metalen = xdp->data - xdp->data_meta;
+ from_buf = from->data_meta;
+ metalen = from->data - from->data_meta;
+ to_buf = to->data - metalen;
}
- memcpy_addr = xsk_umem_adjust_offset(xs->umem, addr, offset);
- __xsk_rcv_memcpy(xs->umem, memcpy_addr, from_buf, len, metalen);
-
- offset += metalen;
- addr = xsk_umem_adjust_offset(xs->umem, addr, offset);
- err = xskq_prod_reserve_desc(xs->rx, addr, len);
- if (!err) {
- xskq_cons_release(xs->umem->fq);
- xdp_return_buff(xdp);
- return 0;
- }
-
- xs->rx_dropped++;
- return err;
+ memcpy(to_buf, from_buf, len + metalen);
}
-static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
+static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,
+ bool explicit_free)
{
- int err = xskq_prod_reserve_desc(xs->rx, xdp->handle, len);
+ struct xdp_buff *xsk_xdp;
+ int err;
- if (err)
+ if (len > xsk_umem_get_rx_frame_size(xs->umem)) {
+ xs->rx_dropped++;
+ return -ENOSPC;
+ }
+
+ xsk_xdp = xsk_buff_alloc(xs->umem);
+ if (!xsk_xdp) {
xs->rx_dropped++;
+ return -ENOSPC;
+ }
- return err;
+ xsk_copy_xdp(xsk_xdp, xdp, len);
+ err = __xsk_rcv_zc(xs, xsk_xdp, len);
+ if (err) {
+ xsk_buff_free(xsk_xdp);
+ return err;
+ }
+ if (explicit_free)
+ xdp_return_buff(xdp);
+ return 0;
}
static bool xsk_is_bound(struct xdp_sock *xs)
@@ -199,7 +190,8 @@ static bool xsk_is_bound(struct xdp_sock *xs)
return false;
}
-static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,
+ bool explicit_free)
{
u32 len;
@@ -211,8 +203,10 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
len = xdp->data_end - xdp->data;
- return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
- __xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
+ return xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
+ xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
+ __xsk_rcv_zc(xs, xdp, len) :
+ __xsk_rcv(xs, xdp, len, explicit_free);
}
static void xsk_flush(struct xdp_sock *xs)
@@ -224,46 +218,11 @@ static void xsk_flush(struct xdp_sock *xs)
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
- u32 metalen = xdp->data - xdp->data_meta;
- u32 len = xdp->data_end - xdp->data;
- u64 offset = xs->umem->headroom;
- void *buffer;
- u64 addr;
int err;
spin_lock_bh(&xs->rx_lock);
-
- if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) {
- err = -EINVAL;
- goto out_unlock;
- }
-
- if (!xskq_cons_peek_addr(xs->umem->fq, &addr, xs->umem) ||
- len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
- err = -ENOSPC;
- goto out_drop;
- }
-
- addr = xsk_umem_adjust_offset(xs->umem, addr, offset);
- buffer = xdp_umem_get_data(xs->umem, addr);
- memcpy(buffer, xdp->data_meta, len + metalen);
-
- addr = xsk_umem_adjust_offset(xs->umem, addr, metalen);
- err = xskq_prod_reserve_desc(xs->rx, addr, len);
- if (err)
- goto out_drop;
-
- xskq_cons_release(xs->umem->fq);
- xskq_prod_submit(xs->rx);
-
- spin_unlock_bh(&xs->rx_lock);
-
- xs->sk.sk_data_ready(&xs->sk);
- return 0;
-
-out_drop:
- xs->rx_dropped++;
-out_unlock:
+ err = xsk_rcv(xs, xdp, false);
+ xsk_flush(xs);
spin_unlock_bh(&xs->rx_lock);
return err;
}
@@ -273,7 +232,7 @@ int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
int err;
- err = xsk_rcv(xs, xdp);
+ err = xsk_rcv(xs, xdp, true);
if (err)
return err;
@@ -404,7 +363,7 @@ static int xsk_generic_xmit(struct sock *sk)
skb_put(skb, len);
addr = desc.addr;
- buffer = xdp_umem_get_data(xs->umem, addr);
+ buffer = xsk_buff_raw_get_data(xs->umem, addr);
err = skb_store_bits(skb, 0, buffer, len);
/* This is the backpressure mechanism for the Tx path.
* Reserve space in the completion queue and only proceed
@@ -860,6 +819,8 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
&xs->umem->cq;
err = xsk_init_queue(entries, q, true);
+ if (optname == XDP_UMEM_FILL_RING)
+ xp_set_fq(xs->umem->pool, *q);
mutex_unlock(&xs->mutex);
return err;
}
diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
new file mode 100644
index 000000000000..df5db2c38859
--- /dev/null
+++ b/net/xdp/xsk_buff_pool.c
@@ -0,0 +1,462 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <net/xsk_buff_pool.h>
+#include <net/xdp_sock.h>
+#include <linux/dma-direct.h>
+#include <linux/dma-noncoherent.h>
+#include <linux/swiotlb.h>
+
+#include "xsk_queue.h"
+
+struct xsk_buff_pool {
+ struct xsk_queue *fq;
+ struct list_head free_list;
+ dma_addr_t *dma_pages;
+ struct xdp_buff_xsk *heads;
+ u64 chunk_mask;
+ u64 addrs_cnt;
+ u32 free_list_cnt;
+ u32 dma_pages_cnt;
+ u32 heads_cnt;
+ u32 free_heads_cnt;
+ u32 headroom;
+ u32 chunk_size;
+ u32 frame_len;
+ bool cheap_dma;
+ bool unaligned;
+ void *addrs;
+ struct device *dev;
+ struct xdp_buff_xsk *free_heads[];
+};
+
+static void xp_addr_unmap(struct xsk_buff_pool *pool)
+{
+ vunmap(pool->addrs);
+}
+
+static int xp_addr_map(struct xsk_buff_pool *pool,
+ struct page **pages, u32 nr_pages)
+{
+ pool->addrs = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
+ if (!pool->addrs)
+ return -ENOMEM;
+ return 0;
+}
+
+void xp_destroy(struct xsk_buff_pool *pool)
+{
+ if (!pool)
+ return;
+
+ xp_addr_unmap(pool);
+ kvfree(pool->heads);
+ kvfree(pool);
+}
+
+struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
+ u32 chunk_size, u32 headroom, u64 size,
+ bool unaligned)
+{
+ struct xsk_buff_pool *pool;
+ struct xdp_buff_xsk *xskb;
+ int err;
+ u32 i;
+
+ pool = kvzalloc(struct_size(pool, free_heads, chunks), GFP_KERNEL);
+ if (!pool)
+ goto out;
+
+ pool->heads = kvcalloc(chunks, sizeof(*pool->heads), GFP_KERNEL);
+ if (!pool->heads)
+ goto out;
+
+ pool->chunk_mask = ~((u64)chunk_size - 1);
+ pool->addrs_cnt = size;
+ pool->heads_cnt = chunks;
+ pool->free_heads_cnt = chunks;
+ pool->headroom = headroom;
+ pool->chunk_size = chunk_size;
+ pool->cheap_dma = true;
+ pool->unaligned = unaligned;
+ pool->frame_len = chunk_size - headroom - XDP_PACKET_HEADROOM;
+ INIT_LIST_HEAD(&pool->free_list);
+
+ for (i = 0; i < pool->free_heads_cnt; i++) {
+ xskb = &pool->heads[i];
+ xskb->pool = pool;
+ pool->free_heads[i] = xskb;
+ }
+
+ err = xp_addr_map(pool, pages, nr_pages);
+ if (!err)
+ return pool;
+
+out:
+ xp_destroy(pool);
+ return NULL;
+}
+
+void xp_set_fq(struct xsk_buff_pool *pool, struct xsk_queue *fq)
+{
+ pool->fq = fq;
+}
+
+void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
+{
+ u32 i;
+
+ for (i = 0; i < pool->heads_cnt; i++)
+ pool->heads[i].xdp.rxq = rxq;
+}
+EXPORT_SYMBOL(xp_set_rxq_info);
+
+void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
+{
+ dma_addr_t *dma;
+ u32 i;
+
+ if (pool->dma_pages_cnt == 0)
+ return;
+
+ for (i = 0; i < pool->dma_pages_cnt; i++) {
+ dma = &pool->dma_pages[i];
+ if (*dma) {
+ dma_unmap_page_attrs(pool->dev, *dma, PAGE_SIZE,
+ DMA_BIDIRECTIONAL, attrs);
+ *dma = 0;
+ }
+ }
+
+ kvfree(pool->dma_pages);
+ pool->dma_pages_cnt = 0;
+ pool->dev = NULL;
+}
+EXPORT_SYMBOL(xp_dma_unmap);
+
+static void xp_check_dma_contiguity(struct xsk_buff_pool *pool)
+{
+ u32 i;
+
+ for (i = 0; i < pool->dma_pages_cnt - 1; i++) {
+ if (pool->dma_pages[i] + PAGE_SIZE == pool->dma_pages[i + 1])
+ pool->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
+ else
+ pool->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
+ }
+}
+
+static bool __maybe_unused xp_check_swiotlb_dma(struct xsk_buff_pool *pool)
+{
+#if defined(CONFIG_SWIOTLB)
+ phys_addr_t paddr;
+ u32 i;
+
+ for (i = 0; i < pool->dma_pages_cnt; i++) {
+ paddr = dma_to_phys(pool->dev, pool->dma_pages[i]);
+ if (is_swiotlb_buffer(paddr))
+ return false;
+ }
+#endif
+ return true;
+}
+
+static bool xp_check_cheap_dma(struct xsk_buff_pool *pool)
+{
+#if defined(CONFIG_HAS_DMA)
+ const struct dma_map_ops *ops = get_dma_ops(pool->dev);
+
+ if (ops) {
+ return !ops->sync_single_for_cpu &&
+ !ops->sync_single_for_device;
+ }
+
+ if (!dma_is_direct(ops))
+ return false;
+
+ if (!xp_check_swiotlb_dma(pool))
+ return false;
+
+ if (!dev_is_dma_coherent(pool->dev)) {
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
+ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
+ return false;
+#endif
+ }
+#endif
+ return true;
+}
+
+int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
+ unsigned long attrs, struct page **pages, u32 nr_pages)
+{
+ dma_addr_t dma;
+ u32 i;
+
+ pool->dma_pages = kvcalloc(nr_pages, sizeof(*pool->dma_pages),
+ GFP_KERNEL);
+ if (!pool->dma_pages)
+ return -ENOMEM;
+
+ pool->dev = dev;
+ pool->dma_pages_cnt = nr_pages;
+
+ for (i = 0; i < pool->dma_pages_cnt; i++) {
+ dma = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL, attrs);
+ if (dma_mapping_error(dev, dma)) {
+ xp_dma_unmap(pool, attrs);
+ return -ENOMEM;
+ }
+ pool->dma_pages[i] = dma;
+ }
+
+ if (pool->unaligned)
+ xp_check_dma_contiguity(pool);
+
+ pool->dev = dev;
+ pool->cheap_dma = xp_check_cheap_dma(pool);
+ return 0;
+}
+EXPORT_SYMBOL(xp_dma_map);
+
+static bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,
+ u64 addr, u32 len)
+{
+ bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE;
+
+ if (pool->dma_pages_cnt && cross_pg) {
+ return !(pool->dma_pages[addr >> PAGE_SHIFT] &
+ XSK_NEXT_PG_CONTIG_MASK);
+ }
+ return false;
+}
+
+static bool xp_addr_crosses_non_contig_pg(struct xsk_buff_pool *pool,
+ u64 addr)
+{
+ return xp_desc_crosses_non_contig_pg(pool, addr, pool->chunk_size);
+}
+
+void xp_release(struct xdp_buff_xsk *xskb)
+{
+ xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;
+}
+
+static u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)
+{
+ return addr & pool->chunk_mask;
+}
+
+static u64 xp_unaligned_extract_addr(u64 addr)
+{
+ return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
+}
+
+static u64 xp_unaligned_extract_offset(u64 addr)
+{
+ return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
+}
+
+static u64 xp_unaligned_add_offset_to_addr(u64 addr)
+{
+ return xp_unaligned_extract_addr(addr) +
+ xp_unaligned_extract_offset(addr);
+}
+
+static bool xp_check_unaligned(struct xsk_buff_pool *pool, u64 *addr)
+{
+ *addr = xp_unaligned_extract_addr(*addr);
+ if (*addr >= pool->addrs_cnt ||
+ *addr + pool->chunk_size > pool->addrs_cnt ||
+ xp_addr_crosses_non_contig_pg(pool, *addr))
+ return false;
+ return true;
+}
+
+static bool xp_check_aligned(struct xsk_buff_pool *pool, u64 *addr)
+{
+ *addr = xp_aligned_extract_addr(pool, *addr);
+ return *addr < pool->addrs_cnt;
+}
+
+static struct xdp_buff_xsk *__xp_alloc(struct xsk_buff_pool *pool)
+{
+ struct xdp_buff_xsk *xskb;
+ u64 addr;
+ bool ok;
+
+ if (pool->free_heads_cnt == 0)
+ return NULL;
+
+ xskb = pool->free_heads[--pool->free_heads_cnt];
+
+ for (;;) {
+ if (!xskq_cons_peek_addr_unchecked(pool->fq, &addr)) {
+ xp_release(xskb);
+ return NULL;
+ }
+
+ ok = pool->unaligned ? xp_check_unaligned(pool, &addr) :
+ xp_check_aligned(pool, &addr);
+ if (!ok) {
+ pool->fq->invalid_descs++;
+ xskq_cons_release(pool->fq);
+ continue;
+ }
+ break;
+ }
+ xskq_cons_release(pool->fq);
+
+ xskb->orig_addr = addr;
+ xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom;
+ if (pool->dma_pages_cnt) {
+ xskb->dma = (pool->dma_pages[addr >> PAGE_SHIFT] &
+ ~XSK_NEXT_PG_CONTIG_MASK) +
+ (addr & ~PAGE_MASK) +
+ pool->headroom + XDP_PACKET_HEADROOM;
+ }
+ return xskb;
+}
+
+struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool)
+{
+ struct xdp_buff_xsk *xskb;
+
+ if (!pool->free_list_cnt) {
+ xskb = __xp_alloc(pool);
+ if (!xskb)
+ return NULL;
+ } else {
+ pool->free_list_cnt--;
+ xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk,
+ free_list_node);
+ list_del(&xskb->free_list_node);
+ }
+
+ xskb->xdp.data = xskb->xdp.data_hard_start + XDP_PACKET_HEADROOM;
+ xskb->xdp.data_meta = xskb->xdp.data;
+
+ if (!pool->cheap_dma) {
+ dma_sync_single_range_for_device(pool->dev, xskb->dma, 0,
+ pool->frame_len,
+ DMA_BIDIRECTIONAL);
+ }
+ return &xskb->xdp;
+}
+EXPORT_SYMBOL(xp_alloc);
+
+bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count)
+{
+ if (pool->free_list_cnt >= count)
+ return true;
+ return xskq_cons_has_entries(pool->fq, count - pool->free_list_cnt);
+}
+EXPORT_SYMBOL(xp_can_alloc);
+
+void xp_free(struct xdp_buff_xsk *xskb)
+{
+ xskb->pool->free_list_cnt++;
+ list_add(&xskb->free_list_node, &xskb->pool->free_list);
+}
+EXPORT_SYMBOL(xp_free);
+
+static bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ u64 chunk, chunk_end;
+
+ chunk = xp_aligned_extract_addr(pool, desc->addr);
+ chunk_end = xp_aligned_extract_addr(pool, desc->addr + desc->len);
+ if (chunk != chunk_end)
+ return false;
+
+ if (chunk >= pool->addrs_cnt)
+ return false;
+
+ if (desc->options)
+ return false;
+ return true;
+}
+
+static bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ u64 addr, base_addr;
+
+ base_addr = xp_unaligned_extract_addr(desc->addr);
+ addr = xp_unaligned_add_offset_to_addr(desc->addr);
+
+ if (desc->len > pool->chunk_size)
+ return false;
+
+ if (base_addr >= pool->addrs_cnt || addr >= pool->addrs_cnt ||
+ xp_desc_crosses_non_contig_pg(pool, addr, desc->len))
+ return false;
+
+ if (desc->options)
+ return false;
+ return true;
+}
+
+bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
+{
+ return pool->unaligned ? xp_unaligned_validate_desc(pool, desc) :
+ xp_aligned_validate_desc(pool, desc);
+}
+
+u64 xp_get_handle(struct xdp_buff_xsk *xskb)
+{
+ u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;
+
+ offset += xskb->pool->headroom;
+ if (!xskb->pool->unaligned)
+ return xskb->orig_addr + offset;
+ return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
+}
+
+void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
+{
+ addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
+ return pool->addrs + addr;
+}
+EXPORT_SYMBOL(xp_raw_get_data);
+
+dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr)
+{
+ addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
+ return (pool->dma_pages[addr >> PAGE_SHIFT] &
+ ~XSK_NEXT_PG_CONTIG_MASK) +
+ (addr & ~PAGE_MASK);
+}
+EXPORT_SYMBOL(xp_raw_get_dma);
+
+dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb)
+{
+ return xskb->dma;
+}
+EXPORT_SYMBOL(xp_get_dma);
+
+void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb)
+{
+ size_t size;
+
+ if (xskb->pool->cheap_dma)
+ return;
+
+ size = xskb->xdp.data_end - xskb->xdp.data;
+ dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0,
+ size, DMA_BIDIRECTIONAL);
+}
+EXPORT_SYMBOL(xp_dma_sync_for_cpu);
+
+void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
+ size_t size)
+{
+ if (pool->cheap_dma)
+ return;
+
+ dma_sync_single_range_for_device(pool->dev, dma, 0,
+ size, DMA_BIDIRECTIONAL);
+}
+EXPORT_SYMBOL(xp_dma_sync_for_device);
diff --git a/net/xdp/xsk_diag.c b/net/xdp/xsk_diag.c
index f59791ba43a0..0163b26aaf63 100644
--- a/net/xdp/xsk_diag.c
+++ b/net/xdp/xsk_diag.c
@@ -56,7 +56,7 @@ static int xsk_diag_put_umem(const struct xdp_sock *xs, struct sk_buff *nlskb)
du.id = umem->id;
du.size = umem->size;
du.num_pages = umem->npgs;
- du.chunk_size = umem->chunk_size_nohr + umem->headroom;
+ du.chunk_size = umem->chunk_size;
du.headroom = umem->headroom;
du.ifindex = umem->dev ? umem->dev->ifindex : 0;
du.queue_id = umem->queue_id;
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index a322a7dac58c..9151aef7dbca 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -9,6 +9,7 @@
#include <linux/types.h>
#include <linux/if_xdp.h>
#include <net/xdp_sock.h>
+#include <net/xsk_buff_pool.h>
#include "xsk.h"
@@ -172,31 +173,45 @@ static inline bool xskq_cons_read_addr(struct xsk_queue *q, u64 *addr,
return false;
}
-static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
- struct xdp_desc *d,
- struct xdp_umem *umem)
+static inline bool xskq_cons_read_addr_aligned(struct xsk_queue *q, u64 *addr)
{
- if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG) {
- if (!xskq_cons_is_valid_unaligned(q, d->addr, d->len, umem))
- return false;
+ struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
- if (d->len > umem->chunk_size_nohr || d->options) {
- q->invalid_descs++;
- return false;
- }
+ while (q->cached_cons != q->cached_prod) {
+ u32 idx = q->cached_cons & q->ring_mask;
+
+ *addr = ring->desc[idx];
+ if (xskq_cons_is_valid_addr(q, *addr))
+ return true;
+ q->cached_cons++;
+ }
+
+ return false;
+}
+
+static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr)
+{
+ struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
+
+ if (q->cached_cons != q->cached_prod) {
+ u32 idx = q->cached_cons & q->ring_mask;
+
+ *addr = ring->desc[idx];
return true;
}
- if (!xskq_cons_is_valid_addr(q, d->addr))
- return false;
+ return false;
+}
- if (((d->addr + d->len) & q->chunk_mask) != (d->addr & q->chunk_mask) ||
- d->options) {
+static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
+ struct xdp_desc *d,
+ struct xdp_umem *umem)
+{
+ if (!xp_validate_desc(umem->pool, d)) {
q->invalid_descs++;
return false;
}
-
return true;
}
@@ -260,6 +275,20 @@ static inline bool xskq_cons_peek_addr(struct xsk_queue *q, u64 *addr,
return xskq_cons_read_addr(q, addr, umem);
}
+static inline bool xskq_cons_peek_addr_aligned(struct xsk_queue *q, u64 *addr)
+{
+ if (q->cached_prod == q->cached_cons)
+ xskq_cons_get_entries(q);
+ return xskq_cons_read_addr_aligned(q, addr);
+}
+
+static inline bool xskq_cons_peek_addr_unchecked(struct xsk_queue *q, u64 *addr)
+{
+ if (q->cached_prod == q->cached_cons)
+ xskq_cons_get_entries(q);
+ return xskq_cons_read_addr_unchecked(q, addr);
+}
+
static inline bool xskq_cons_peek_desc(struct xsk_queue *q,
struct xdp_desc *desc,
struct xdp_umem *umem)
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [PATCH bpf-next 04/14] xsk: introduce AF_XDP buffer allocation API
2020-05-07 10:42 ` [PATCH bpf-next 04/14] xsk: introduce AF_XDP buffer allocation API Björn Töpel
@ 2020-05-08 11:55 ` Maxim Mikityanskiy
2020-05-08 12:32 ` Björn Töpel
0 siblings, 1 reply; 24+ messages in thread
From: Maxim Mikityanskiy @ 2020-05-08 11:55 UTC (permalink / raw)
To: Björn Töpel, Björn Töpel
Cc: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher,
maciej.fijalkowski
On 2020-05-07 13:42, Björn Töpel wrote:
> From: Björn Töpel <bjorn.topel@intel.com>
>
> In order to simplify AF_XDP zero-copy enablement for NIC driver
> developers, a new AF_XDP buffer allocation API is added. The
> implementation is based on a single core (single producer/consumer)
> buffer pool for the AF_XDP UMEM.
>
> A buffer is allocated using the xsk_buff_alloc() function, and
> returned using xsk_buff_free(). If a buffer is disassociated with the
> pool, e.g. when a buffer is passed to an AF_XDP socket, a buffer is
> said to be released. Currently, the release function is only used by
> the AF_XDP internals and not visible to the driver.
>
> Drivers using this API should register the XDP memory model with the
> new MEM_TYPE_XSK_BUFF_POOL type.
>
> The API is defined in net/xdp_sock_drv.h.
>
> The buffer type is struct xdp_buff, and follows the lifetime of
> regular xdp_buffs, i.e. the lifetime of an xdp_buff is restricted to
> a NAPI context. In other words, the API is not replacing xdp_frames.
>
> In addition to introducing the API and implementations, the AF_XDP
> core is migrated to use the new APIs.
>
> rfc->v1: Fixed build errors/warnings for m68k and riscv. (kbuild test
> robot)
> Added headroom/chunk size getter. (Maxim/Björn)
>
> Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
> Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Björn's signoff should go first (as the original author).
> ---
> include/net/xdp.h | 4 +-
> include/net/xdp_sock.h | 2 +
> include/net/xdp_sock_drv.h | 152 ++++++++++++
> include/net/xsk_buff_pool.h | 54 +++++
> include/trace/events/xdp.h | 3 +-
> net/core/xdp.c | 14 +-
> net/xdp/Makefile | 1 +
> net/xdp/xdp_umem.c | 19 +-
> net/xdp/xsk.c | 147 +++++-------
> net/xdp/xsk_buff_pool.c | 462 ++++++++++++++++++++++++++++++++++++
> net/xdp/xsk_diag.c | 2 +-
> net/xdp/xsk_queue.h | 59 +++--
> 12 files changed, 800 insertions(+), 119 deletions(-)
> create mode 100644 include/net/xsk_buff_pool.h
> create mode 100644 net/xdp/xsk_buff_pool.c
>
> diff --git a/include/net/xdp.h b/include/net/xdp.h
> index 3cc6d5d84aa4..83173e4d306c 100644
> --- a/include/net/xdp.h
> +++ b/include/net/xdp.h
> @@ -38,6 +38,7 @@ enum xdp_mem_type {
> MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
> MEM_TYPE_PAGE_POOL,
> MEM_TYPE_ZERO_COPY,
> + MEM_TYPE_XSK_BUFF_POOL,
> MEM_TYPE_MAX,
> };
>
> @@ -101,7 +102,8 @@ struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
> int metasize;
> int headroom;
>
> - if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
> + if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
> + xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
> return xdp_convert_zc_to_xdp_frame(xdp);
>
> /* Assure headroom is available for storing info */
> diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
> index fb7fe3060175..6e7265f63c04 100644
> --- a/include/net/xdp_sock.h
> +++ b/include/net/xdp_sock.h
> @@ -31,11 +31,13 @@ struct xdp_umem_fq_reuse {
> struct xdp_umem {
> struct xsk_queue *fq;
> struct xsk_queue *cq;
> + struct xsk_buff_pool *pool;
> struct xdp_umem_page *pages;
> u64 chunk_mask;
> u64 size;
> u32 headroom;
> u32 chunk_size_nohr;
> + u32 chunk_size;
> struct user_struct *user;
> refcount_t users;
> struct work_struct work;
> diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
> index 98dd6962e6d4..5a0970d4c44c 100644
> --- a/include/net/xdp_sock_drv.h
> +++ b/include/net/xdp_sock_drv.h
> @@ -7,6 +7,7 @@
> #define _LINUX_XDP_SOCK_DRV_H
>
> #include <net/xdp_sock.h>
> +#include <net/xsk_buff_pool.h>
>
> #ifdef CONFIG_XDP_SOCKETS
>
> @@ -96,6 +97,87 @@ static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 address,
> return address + offset;
> }
>
> +static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
> +{
> + return XDP_PACKET_HEADROOM + umem->headroom;
> +}
> +
> +static inline u32 xsk_umem_get_chunk_size(struct xdp_umem *umem)
> +{
> + return umem->chunk_size;
> +}
> +
> +static inline u32 xsk_umem_get_rx_frame_size(struct xdp_umem *umem)
> +{
> + return xsk_umem_get_chunk_size(umem) - xsk_umem_get_headroom(umem);
> +}
> +
> +static inline void xsk_buff_set_rxq_info(struct xdp_umem *umem,
> + struct xdp_rxq_info *rxq)
> +{
> + xp_set_rxq_info(umem->pool, rxq);
> +}
> +
> +static inline void xsk_buff_dma_unmap(struct xdp_umem *umem,
> + unsigned long attrs)
> +{
> + xp_dma_unmap(umem->pool, attrs);
> +}
> +
> +static inline int xsk_buff_dma_map(struct xdp_umem *umem, struct device *dev,
> + unsigned long attrs)
> +{
> + return xp_dma_map(umem->pool, dev, attrs, umem->pgs, umem->npgs);
> +}
> +
> +static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
> +{
> + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
> +
> + return xp_get_dma(xskb);
> +}
> +
> +static inline struct xdp_buff *xsk_buff_alloc(struct xdp_umem *umem)
> +{
> + return xp_alloc(umem->pool);
> +}
> +
> +static inline bool xsk_buff_can_alloc(struct xdp_umem *umem, u32 count)
> +{
> + return xp_can_alloc(umem->pool, count);
> +}
> +
> +static inline void xsk_buff_free(struct xdp_buff *xdp)
> +{
> + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
> +
> + xp_free(xskb);
> +}
> +
> +static inline dma_addr_t xsk_buff_raw_get_dma(struct xdp_umem *umem, u64 addr)
> +{
> + return xp_raw_get_dma(umem->pool, addr);
> +}
> +
> +static inline void *xsk_buff_raw_get_data(struct xdp_umem *umem, u64 addr)
> +{
> + return xp_raw_get_data(umem->pool, addr);
> +}
> +
> +static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
> +{
> + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
> +
> + xp_dma_sync_for_cpu(xskb);
> +}
> +
> +static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
> + dma_addr_t dma,
> + size_t size)
> +{
> + xp_dma_sync_for_device(umem->pool, dma, size);
> +}
> +
> #else
>
> static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
> @@ -202,6 +284,76 @@ static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 handle,
> return 0;
> }
>
> +static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
> +{
> + return 0;
> +}
> +
> +static inline u32 xsk_umem_get_chunk_size(struct xdp_umem *umem)
> +{
> + return 0;
> +}
> +
> +static inline u32 xsk_umem_get_rx_frame_size(struct xdp_umem *umem)
> +{
> + return 0;
> +}
> +
> +static inline void xsk_buff_set_rxq_info(struct xdp_umem *umem,
> + struct xdp_rxq_info *rxq)
> +{
> +}
> +
> +static inline void xsk_buff_dma_unmap(struct xdp_umem *umem,
> + unsigned long attrs)
> +{
> +}
> +
> +static inline int xsk_buff_dma_map(struct xdp_umem *umem, struct device *dev,
> + unsigned long attrs)
> +{
> + return 0;
> +}
> +
> +static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
> +{
> + return 0;
> +}
> +
> +static inline struct xdp_buff *xsk_buff_alloc(struct xdp_umem *umem)
> +{
> + return NULL;
> +}
> +
> +static inline bool xsk_buff_can_alloc(struct xdp_umem *umem, u32 count)
> +{
> + return false;
> +}
> +
> +static inline void xsk_buff_free(struct xdp_buff *xdp)
> +{
> +}
> +
> +static inline dma_addr_t xsk_buff_raw_get_dma(struct xdp_umem *umem, u64 addr)
> +{
> + return 0;
> +}
> +
> +static inline void *xsk_buff_raw_get_data(struct xdp_umem *umem, u64 addr)
> +{
> + return NULL;
> +}
> +
> +static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
> +{
> +}
> +
> +static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
> + dma_addr_t dma,
> + size_t size)
> +{
> +}
> +
> #endif /* CONFIG_XDP_SOCKETS */
>
> #endif /* _LINUX_XDP_SOCK_DRV_H */
> diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
> new file mode 100644
> index 000000000000..9abef166441d
> --- /dev/null
> +++ b/include/net/xsk_buff_pool.h
> @@ -0,0 +1,54 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2020 Intel Corporation. */
> +
> +#ifndef XSK_BUFF_POOL_H_
> +#define XSK_BUFF_POOL_H_
> +
> +#include <linux/types.h>
> +#include <linux/dma-mapping.h>
> +#include <net/xdp.h>
> +
> +struct xsk_buff_pool;
> +struct xdp_rxq_info;
> +struct xsk_queue;
> +struct xdp_desc;
> +struct device;
> +struct page;
> +
> +struct xdp_buff_xsk {
> + struct xdp_buff xdp;
> + dma_addr_t dma;
> + struct xsk_buff_pool *pool;
> + bool unaligned;
> + u64 orig_addr;
> + struct list_head free_list_node;
> +};
> +
> +/* AF_XDP core. */
> +struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
> + u32 chunk_size, u32 headroom, u64 size,
> + bool unaligned);
> +void xp_set_fq(struct xsk_buff_pool *pool, struct xsk_queue *fq);
> +void xp_destroy(struct xsk_buff_pool *pool);
> +void xp_release(struct xdp_buff_xsk *xskb);
> +u64 xp_get_handle(struct xdp_buff_xsk *xskb);
> +bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
> +
> +/* AF_XDP, and XDP core. */
> +void xp_free(struct xdp_buff_xsk *xskb);
> +
> +/* AF_XDP ZC drivers, via xdp_sock_buff.h */
> +void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq);
> +int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
> + unsigned long attrs, struct page **pages, u32 nr_pages);
> +void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs);
> +struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
> +bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
> +void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
> +dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
> +dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb);
> +void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb);
> +void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
> + size_t size);
> +
> +#endif /* XSK_BUFF_POOL_H_ */
> diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
> index b95d65e8c628..48547a12fa27 100644
> --- a/include/trace/events/xdp.h
> +++ b/include/trace/events/xdp.h
> @@ -287,7 +287,8 @@ TRACE_EVENT(xdp_devmap_xmit,
> FN(PAGE_SHARED) \
> FN(PAGE_ORDER0) \
> FN(PAGE_POOL) \
> - FN(ZERO_COPY)
> + FN(ZERO_COPY) \
> + FN(XSK_BUFF_POOL)
>
> #define __MEM_TYPE_TP_FN(x) \
> TRACE_DEFINE_ENUM(MEM_TYPE_##x);
> diff --git a/net/core/xdp.c b/net/core/xdp.c
> index 4c7ea85486af..89053ef8333b 100644
> --- a/net/core/xdp.c
> +++ b/net/core/xdp.c
> @@ -16,6 +16,7 @@
> #include <net/xdp.h>
> #include <net/xdp_priv.h> /* struct xdp_mem_allocator */
> #include <trace/events/xdp.h>
> +#include <net/xdp_sock_drv.h>
>
> #define REG_STATE_NEW 0x0
> #define REG_STATE_REGISTERED 0x1
> @@ -360,7 +361,7 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
> * of xdp_frames/pages in those cases.
> */
> static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
> - unsigned long handle)
> + unsigned long handle, struct xdp_buff *xdp)
> {
> struct xdp_mem_allocator *xa;
> struct page *page;
> @@ -389,6 +390,11 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
> xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
> xa->zc_alloc->free(xa->zc_alloc, handle);
> rcu_read_unlock();
> + break;
> + case MEM_TYPE_XSK_BUFF_POOL:
> + /* NB! Only valid from an xdp_buff! */
> + xsk_buff_free(xdp);
> + break;
> default:
> /* Not possible, checked in xdp_rxq_info_reg_mem_model() */
> break;
> @@ -397,19 +403,19 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
>
> void xdp_return_frame(struct xdp_frame *xdpf)
> {
> - __xdp_return(xdpf->data, &xdpf->mem, false, 0);
> + __xdp_return(xdpf->data, &xdpf->mem, false, 0, NULL);
> }
> EXPORT_SYMBOL_GPL(xdp_return_frame);
>
> void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
> {
> - __xdp_return(xdpf->data, &xdpf->mem, true, 0);
> + __xdp_return(xdpf->data, &xdpf->mem, true, 0, NULL);
> }
> EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
>
> void xdp_return_buff(struct xdp_buff *xdp)
> {
> - __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle);
> + __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle, xdp);
> }
> EXPORT_SYMBOL_GPL(xdp_return_buff);
>
> diff --git a/net/xdp/Makefile b/net/xdp/Makefile
> index 90b5460d6166..30cdc4315f42 100644
> --- a/net/xdp/Makefile
> +++ b/net/xdp/Makefile
> @@ -1,3 +1,4 @@
> # SPDX-License-Identifier: GPL-2.0-only
> obj-$(CONFIG_XDP_SOCKETS) += xsk.o xdp_umem.o xsk_queue.o xskmap.o
> +obj-$(CONFIG_XDP_SOCKETS) += xsk_buff_pool.o
> obj-$(CONFIG_XDP_SOCKETS_DIAG) += xsk_diag.o
> diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
> index 37ace3bc0d48..7f04688045d5 100644
> --- a/net/xdp/xdp_umem.c
> +++ b/net/xdp/xdp_umem.c
> @@ -245,7 +245,7 @@ static void xdp_umem_release(struct xdp_umem *umem)
> }
>
> xsk_reuseq_destroy(umem);
> -
> + xp_destroy(umem->pool);
> xdp_umem_unmap_pages(umem);
> xdp_umem_unpin_pages(umem);
>
> @@ -390,6 +390,7 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
> umem->size = size;
> umem->headroom = headroom;
> umem->chunk_size_nohr = chunk_size - headroom;
> + umem->chunk_size = chunk_size;
> umem->npgs = size / PAGE_SIZE;
> umem->pgs = NULL;
> umem->user = NULL;
> @@ -415,11 +416,21 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
> }
>
> err = xdp_umem_map_pages(umem);
> - if (!err)
> - return 0;
> + if (err)
> + goto out_pages;
>
> - kvfree(umem->pages);
> + umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size,
> + headroom, size, unaligned_chunks);
> + if (!umem->pool) {
> + err = -ENOMEM;
> + goto out_unmap;
> + }
> + return 0;
>
> +out_unmap:
> + xdp_umem_unmap_pages(umem);
> +out_pages:
> + kvfree(umem->pages);
> out_pin:
> xdp_umem_unpin_pages(umem);
> out_account:
> diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
> index 8bda654e82ec..6933f0d494ba 100644
> --- a/net/xdp/xsk.c
> +++ b/net/xdp/xsk.c
> @@ -117,76 +117,67 @@ bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
> }
> EXPORT_SYMBOL(xsk_umem_uses_need_wakeup);
>
> -/* If a buffer crosses a page boundary, we need to do 2 memcpy's, one for
> - * each page. This is only required in copy mode.
> - */
> -static void __xsk_rcv_memcpy(struct xdp_umem *umem, u64 addr, void *from_buf,
> - u32 len, u32 metalen)
> +static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
> {
> - void *to_buf = xdp_umem_get_data(umem, addr);
> -
> - addr = xsk_umem_add_offset_to_addr(addr);
> - if (xskq_cons_crosses_non_contig_pg(umem, addr, len + metalen)) {
> - void *next_pg_addr = umem->pages[(addr >> PAGE_SHIFT) + 1].addr;
> - u64 page_start = addr & ~(PAGE_SIZE - 1);
> - u64 first_len = PAGE_SIZE - (addr - page_start);
> -
> - memcpy(to_buf, from_buf, first_len);
> - memcpy(next_pg_addr, from_buf + first_len,
> - len + metalen - first_len);
> + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
> + u64 addr;
> + int err;
>
> - return;
> + addr = xp_get_handle(xskb);
> + err = xskq_prod_reserve_desc(xs->rx, addr, len);
> + if (err) {
> + xs->rx_dropped++;
> + return err;
> }
>
> - memcpy(to_buf, from_buf, len + metalen);
> + xp_release(xskb);
> + return 0;
> }
>
> -static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
> +static void xsk_copy_xdp(struct xdp_buff *to, struct xdp_buff *from, u32 len)
> {
> - u64 offset = xs->umem->headroom;
> - u64 addr, memcpy_addr;
> - void *from_buf;
> + void *from_buf, *to_buf;
> u32 metalen;
> - int err;
> -
> - if (!xskq_cons_peek_addr(xs->umem->fq, &addr, xs->umem) ||
> - len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
> - xs->rx_dropped++;
> - return -ENOSPC;
> - }
>
> - if (unlikely(xdp_data_meta_unsupported(xdp))) {
> - from_buf = xdp->data;
> + if (unlikely(xdp_data_meta_unsupported(from))) {
> + from_buf = from->data;
> + to_buf = to->data;
> metalen = 0;
> } else {
> - from_buf = xdp->data_meta;
> - metalen = xdp->data - xdp->data_meta;
> + from_buf = from->data_meta;
> + metalen = from->data - from->data_meta;
> + to_buf = to->data - metalen;
> }
>
> - memcpy_addr = xsk_umem_adjust_offset(xs->umem, addr, offset);
> - __xsk_rcv_memcpy(xs->umem, memcpy_addr, from_buf, len, metalen);
> -
> - offset += metalen;
> - addr = xsk_umem_adjust_offset(xs->umem, addr, offset);
> - err = xskq_prod_reserve_desc(xs->rx, addr, len);
> - if (!err) {
> - xskq_cons_release(xs->umem->fq);
> - xdp_return_buff(xdp);
> - return 0;
> - }
> -
> - xs->rx_dropped++;
> - return err;
> + memcpy(to_buf, from_buf, len + metalen);
> }
>
> -static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
> +static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,
> + bool explicit_free)
> {
> - int err = xskq_prod_reserve_desc(xs->rx, xdp->handle, len);
> + struct xdp_buff *xsk_xdp;
> + int err;
>
> - if (err)
> + if (len > xsk_umem_get_rx_frame_size(xs->umem)) {
> + xs->rx_dropped++;
> + return -ENOSPC;
> + }
> +
> + xsk_xdp = xsk_buff_alloc(xs->umem);
> + if (!xsk_xdp) {
> xs->rx_dropped++;
> + return -ENOSPC;
> + }
>
> - return err;
> + xsk_copy_xdp(xsk_xdp, xdp, len);
> + err = __xsk_rcv_zc(xs, xsk_xdp, len);
> + if (err) {
> + xsk_buff_free(xsk_xdp);
> + return err;
> + }
> + if (explicit_free)
> + xdp_return_buff(xdp);
> + return 0;
> }
>
> static bool xsk_is_bound(struct xdp_sock *xs)
> @@ -199,7 +190,8 @@ static bool xsk_is_bound(struct xdp_sock *xs)
> return false;
> }
>
> -static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
> +static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,
> + bool explicit_free)
> {
> u32 len;
>
> @@ -211,8 +203,10 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
>
> len = xdp->data_end - xdp->data;
>
> - return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ?
> - __xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len);
> + return xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
> + xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
> + __xsk_rcv_zc(xs, xdp, len) :
> + __xsk_rcv(xs, xdp, len, explicit_free);
> }
>
> static void xsk_flush(struct xdp_sock *xs)
> @@ -224,46 +218,11 @@ static void xsk_flush(struct xdp_sock *xs)
>
> int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
> {
> - u32 metalen = xdp->data - xdp->data_meta;
> - u32 len = xdp->data_end - xdp->data;
> - u64 offset = xs->umem->headroom;
> - void *buffer;
> - u64 addr;
> int err;
>
> spin_lock_bh(&xs->rx_lock);
> -
> - if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) {
> - err = -EINVAL;
> - goto out_unlock;
> - }
> -
> - if (!xskq_cons_peek_addr(xs->umem->fq, &addr, xs->umem) ||
> - len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) {
> - err = -ENOSPC;
> - goto out_drop;
> - }
> -
> - addr = xsk_umem_adjust_offset(xs->umem, addr, offset);
> - buffer = xdp_umem_get_data(xs->umem, addr);
> - memcpy(buffer, xdp->data_meta, len + metalen);
> -
> - addr = xsk_umem_adjust_offset(xs->umem, addr, metalen);
> - err = xskq_prod_reserve_desc(xs->rx, addr, len);
> - if (err)
> - goto out_drop;
> -
> - xskq_cons_release(xs->umem->fq);
> - xskq_prod_submit(xs->rx);
> -
> - spin_unlock_bh(&xs->rx_lock);
> -
> - xs->sk.sk_data_ready(&xs->sk);
> - return 0;
> -
> -out_drop:
> - xs->rx_dropped++;
> -out_unlock:
> + err = xsk_rcv(xs, xdp, false);
> + xsk_flush(xs);
> spin_unlock_bh(&xs->rx_lock);
> return err;
> }
> @@ -273,7 +232,7 @@ int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
> struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
> int err;
>
> - err = xsk_rcv(xs, xdp);
> + err = xsk_rcv(xs, xdp, true);
> if (err)
> return err;
>
> @@ -404,7 +363,7 @@ static int xsk_generic_xmit(struct sock *sk)
>
> skb_put(skb, len);
> addr = desc.addr;
> - buffer = xdp_umem_get_data(xs->umem, addr);
> + buffer = xsk_buff_raw_get_data(xs->umem, addr);
> err = skb_store_bits(skb, 0, buffer, len);
> /* This is the backpressure mechanism for the Tx path.
> * Reserve space in the completion queue and only proceed
> @@ -860,6 +819,8 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname,
> q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq :
> &xs->umem->cq;
> err = xsk_init_queue(entries, q, true);
> + if (optname == XDP_UMEM_FILL_RING)
> + xp_set_fq(xs->umem->pool, *q);
> mutex_unlock(&xs->mutex);
> return err;
> }
> diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
> new file mode 100644
> index 000000000000..df5db2c38859
> --- /dev/null
> +++ b/net/xdp/xsk_buff_pool.c
> @@ -0,0 +1,462 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +#include <net/xsk_buff_pool.h>
> +#include <net/xdp_sock.h>
> +#include <linux/dma-direct.h>
> +#include <linux/dma-noncoherent.h>
> +#include <linux/swiotlb.h>
> +
> +#include "xsk_queue.h"
> +
> +struct xsk_buff_pool {
> + struct xsk_queue *fq;
> + struct list_head free_list;
> + dma_addr_t *dma_pages;
> + struct xdp_buff_xsk *heads;
> + u64 chunk_mask;
> + u64 addrs_cnt;
> + u32 free_list_cnt;
> + u32 dma_pages_cnt;
> + u32 heads_cnt;
> + u32 free_heads_cnt;
> + u32 headroom;
> + u32 chunk_size;
> + u32 frame_len;
> + bool cheap_dma;
> + bool unaligned;
> + void *addrs;
> + struct device *dev;
> + struct xdp_buff_xsk *free_heads[];
> +};
> +
> +static void xp_addr_unmap(struct xsk_buff_pool *pool)
> +{
> + vunmap(pool->addrs);
> +}
> +
> +static int xp_addr_map(struct xsk_buff_pool *pool,
> + struct page **pages, u32 nr_pages)
> +{
> + pool->addrs = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
> + if (!pool->addrs)
> + return -ENOMEM;
> + return 0;
> +}
> +
> +void xp_destroy(struct xsk_buff_pool *pool)
> +{
> + if (!pool)
> + return;
> +
> + xp_addr_unmap(pool);
> + kvfree(pool->heads);
> + kvfree(pool);
> +}
> +
> +struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
> + u32 chunk_size, u32 headroom, u64 size,
> + bool unaligned)
> +{
> + struct xsk_buff_pool *pool;
> + struct xdp_buff_xsk *xskb;
> + int err;
> + u32 i;
> +
> + pool = kvzalloc(struct_size(pool, free_heads, chunks), GFP_KERNEL);
> + if (!pool)
> + goto out;
> +
> + pool->heads = kvcalloc(chunks, sizeof(*pool->heads), GFP_KERNEL);
> + if (!pool->heads)
> + goto out;
> +
> + pool->chunk_mask = ~((u64)chunk_size - 1);
> + pool->addrs_cnt = size;
> + pool->heads_cnt = chunks;
> + pool->free_heads_cnt = chunks;
> + pool->headroom = headroom;
> + pool->chunk_size = chunk_size;
> + pool->cheap_dma = true;
> + pool->unaligned = unaligned;
> + pool->frame_len = chunk_size - headroom - XDP_PACKET_HEADROOM;
> + INIT_LIST_HEAD(&pool->free_list);
> +
> + for (i = 0; i < pool->free_heads_cnt; i++) {
> + xskb = &pool->heads[i];
> + xskb->pool = pool;
> + pool->free_heads[i] = xskb;
> + }
> +
> + err = xp_addr_map(pool, pages, nr_pages);
> + if (!err)
> + return pool;
> +
> +out:
> + xp_destroy(pool);
> + return NULL;
> +}
> +
> +void xp_set_fq(struct xsk_buff_pool *pool, struct xsk_queue *fq)
> +{
> + pool->fq = fq;
> +}
> +
> +void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
> +{
> + u32 i;
> +
> + for (i = 0; i < pool->heads_cnt; i++)
> + pool->heads[i].xdp.rxq = rxq;
> +}
> +EXPORT_SYMBOL(xp_set_rxq_info);
> +
> +void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
> +{
> + dma_addr_t *dma;
> + u32 i;
> +
> + if (pool->dma_pages_cnt == 0)
> + return;
> +
> + for (i = 0; i < pool->dma_pages_cnt; i++) {
> + dma = &pool->dma_pages[i];
> + if (*dma) {
> + dma_unmap_page_attrs(pool->dev, *dma, PAGE_SIZE,
> + DMA_BIDIRECTIONAL, attrs);
> + *dma = 0;
> + }
> + }
> +
> + kvfree(pool->dma_pages);
> + pool->dma_pages_cnt = 0;
> + pool->dev = NULL;
> +}
> +EXPORT_SYMBOL(xp_dma_unmap);
> +
> +static void xp_check_dma_contiguity(struct xsk_buff_pool *pool)
> +{
> + u32 i;
> +
> + for (i = 0; i < pool->dma_pages_cnt - 1; i++) {
> + if (pool->dma_pages[i] + PAGE_SIZE == pool->dma_pages[i + 1])
> + pool->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK;
> + else
> + pool->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK;
> + }
> +}
> +
> +static bool __maybe_unused xp_check_swiotlb_dma(struct xsk_buff_pool *pool)
> +{
> +#if defined(CONFIG_SWIOTLB)
> + phys_addr_t paddr;
> + u32 i;
> +
> + for (i = 0; i < pool->dma_pages_cnt; i++) {
> + paddr = dma_to_phys(pool->dev, pool->dma_pages[i]);
> + if (is_swiotlb_buffer(paddr))
> + return false;
> + }
> +#endif
> + return true;
> +}
> +
> +static bool xp_check_cheap_dma(struct xsk_buff_pool *pool)
> +{
> +#if defined(CONFIG_HAS_DMA)
> + const struct dma_map_ops *ops = get_dma_ops(pool->dev);
> +
> + if (ops) {
> + return !ops->sync_single_for_cpu &&
> + !ops->sync_single_for_device;
> + }
> +
> + if (!dma_is_direct(ops))
> + return false;
> +
> + if (!xp_check_swiotlb_dma(pool))
> + return false;
> +
> + if (!dev_is_dma_coherent(pool->dev)) {
> +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
> + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
> + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
> + return false;
> +#endif
> + }
> +#endif
> + return true;
> +}
> +
> +int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
> + unsigned long attrs, struct page **pages, u32 nr_pages)
> +{
> + dma_addr_t dma;
> + u32 i;
> +
> + pool->dma_pages = kvcalloc(nr_pages, sizeof(*pool->dma_pages),
> + GFP_KERNEL);
> + if (!pool->dma_pages)
> + return -ENOMEM;
> +
> + pool->dev = dev;
> + pool->dma_pages_cnt = nr_pages;
> +
> + for (i = 0; i < pool->dma_pages_cnt; i++) {
> + dma = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE,
> + DMA_BIDIRECTIONAL, attrs);
> + if (dma_mapping_error(dev, dma)) {
> + xp_dma_unmap(pool, attrs);
> + return -ENOMEM;
> + }
> + pool->dma_pages[i] = dma;
> + }
> +
> + if (pool->unaligned)
> + xp_check_dma_contiguity(pool);
> +
> + pool->dev = dev;
> + pool->cheap_dma = xp_check_cheap_dma(pool);
> + return 0;
> +}
> +EXPORT_SYMBOL(xp_dma_map);
> +
> +static bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,
> + u64 addr, u32 len)
> +{
> + bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE;
> +
> + if (pool->dma_pages_cnt && cross_pg) {
> + return !(pool->dma_pages[addr >> PAGE_SHIFT] &
> + XSK_NEXT_PG_CONTIG_MASK);
> + }
> + return false;
> +}
> +
> +static bool xp_addr_crosses_non_contig_pg(struct xsk_buff_pool *pool,
> + u64 addr)
> +{
> + return xp_desc_crosses_non_contig_pg(pool, addr, pool->chunk_size);
> +}
> +
> +void xp_release(struct xdp_buff_xsk *xskb)
> +{
> + xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;
> +}
> +
> +static u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)
> +{
> + return addr & pool->chunk_mask;
> +}
> +
> +static u64 xp_unaligned_extract_addr(u64 addr)
> +{
> + return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
> +}
> +
> +static u64 xp_unaligned_extract_offset(u64 addr)
> +{
> + return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
> +}
> +
> +static u64 xp_unaligned_add_offset_to_addr(u64 addr)
> +{
> + return xp_unaligned_extract_addr(addr) +
> + xp_unaligned_extract_offset(addr);
> +}
> +
> +static bool xp_check_unaligned(struct xsk_buff_pool *pool, u64 *addr)
> +{
> + *addr = xp_unaligned_extract_addr(*addr);
> + if (*addr >= pool->addrs_cnt ||
> + *addr + pool->chunk_size > pool->addrs_cnt ||
> + xp_addr_crosses_non_contig_pg(pool, *addr))
> + return false;
> + return true;
> +}
> +
> +static bool xp_check_aligned(struct xsk_buff_pool *pool, u64 *addr)
> +{
> + *addr = xp_aligned_extract_addr(pool, *addr);
> + return *addr < pool->addrs_cnt;
> +}
> +
> +static struct xdp_buff_xsk *__xp_alloc(struct xsk_buff_pool *pool)
> +{
> + struct xdp_buff_xsk *xskb;
> + u64 addr;
> + bool ok;
> +
> + if (pool->free_heads_cnt == 0)
> + return NULL;
> +
> + xskb = pool->free_heads[--pool->free_heads_cnt];
> +
> + for (;;) {
> + if (!xskq_cons_peek_addr_unchecked(pool->fq, &addr)) {
> + xp_release(xskb);
> + return NULL;
> + }
> +
> + ok = pool->unaligned ? xp_check_unaligned(pool, &addr) :
> + xp_check_aligned(pool, &addr);
> + if (!ok) {
> + pool->fq->invalid_descs++;
> + xskq_cons_release(pool->fq);
> + continue;
> + }
> + break;
> + }
> + xskq_cons_release(pool->fq);
> +
> + xskb->orig_addr = addr;
> + xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom;
> + if (pool->dma_pages_cnt) {
> + xskb->dma = (pool->dma_pages[addr >> PAGE_SHIFT] &
> + ~XSK_NEXT_PG_CONTIG_MASK) +
> + (addr & ~PAGE_MASK) +
> + pool->headroom + XDP_PACKET_HEADROOM;
> + }
> + return xskb;
> +}
> +
> +struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool)
> +{
> + struct xdp_buff_xsk *xskb;
> +
> + if (!pool->free_list_cnt) {
> + xskb = __xp_alloc(pool);
> + if (!xskb)
> + return NULL;
> + } else {
> + pool->free_list_cnt--;
> + xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk,
> + free_list_node);
> + list_del(&xskb->free_list_node);
> + }
> +
> + xskb->xdp.data = xskb->xdp.data_hard_start + XDP_PACKET_HEADROOM;
> + xskb->xdp.data_meta = xskb->xdp.data;
> +
> + if (!pool->cheap_dma) {
> + dma_sync_single_range_for_device(pool->dev, xskb->dma, 0,
> + pool->frame_len,
> + DMA_BIDIRECTIONAL);
> + }
> + return &xskb->xdp;
> +}
> +EXPORT_SYMBOL(xp_alloc);
> +
> +bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count)
> +{
> + if (pool->free_list_cnt >= count)
> + return true;
> + return xskq_cons_has_entries(pool->fq, count - pool->free_list_cnt);
> +}
> +EXPORT_SYMBOL(xp_can_alloc);
> +
> +void xp_free(struct xdp_buff_xsk *xskb)
> +{
> + xskb->pool->free_list_cnt++;
> + list_add(&xskb->free_list_node, &xskb->pool->free_list);
> +}
> +EXPORT_SYMBOL(xp_free);
> +
> +static bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
> + struct xdp_desc *desc)
> +{
> + u64 chunk, chunk_end;
> +
> + chunk = xp_aligned_extract_addr(pool, desc->addr);
> + chunk_end = xp_aligned_extract_addr(pool, desc->addr + desc->len);
> + if (chunk != chunk_end)
> + return false;
> +
> + if (chunk >= pool->addrs_cnt)
> + return false;
> +
> + if (desc->options)
> + return false;
> + return true;
> +}
> +
> +static bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool,
> + struct xdp_desc *desc)
> +{
> + u64 addr, base_addr;
> +
> + base_addr = xp_unaligned_extract_addr(desc->addr);
> + addr = xp_unaligned_add_offset_to_addr(desc->addr);
> +
> + if (desc->len > pool->chunk_size)
> + return false;
> +
> + if (base_addr >= pool->addrs_cnt || addr >= pool->addrs_cnt ||
> + xp_desc_crosses_non_contig_pg(pool, addr, desc->len))
> + return false;
> +
> + if (desc->options)
> + return false;
> + return true;
> +}
> +
> +bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
> +{
> + return pool->unaligned ? xp_unaligned_validate_desc(pool, desc) :
> + xp_aligned_validate_desc(pool, desc);
> +}
> +
> +u64 xp_get_handle(struct xdp_buff_xsk *xskb)
> +{
> + u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;
> +
> + offset += xskb->pool->headroom;
> + if (!xskb->pool->unaligned)
> + return xskb->orig_addr + offset;
> + return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
> +}
> +
> +void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
> +{
> + addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
> + return pool->addrs + addr;
> +}
> +EXPORT_SYMBOL(xp_raw_get_data);
> +
> +dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr)
> +{
> + addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
> + return (pool->dma_pages[addr >> PAGE_SHIFT] &
> + ~XSK_NEXT_PG_CONTIG_MASK) +
> + (addr & ~PAGE_MASK);
> +}
> +EXPORT_SYMBOL(xp_raw_get_dma);
> +
> +dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb)
> +{
> + return xskb->dma;
> +}
> +EXPORT_SYMBOL(xp_get_dma);
> +
> +void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb)
> +{
> + size_t size;
> +
> + if (xskb->pool->cheap_dma)
> + return;
> +
> + size = xskb->xdp.data_end - xskb->xdp.data;
> + dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0,
> + size, DMA_BIDIRECTIONAL);
> +}
> +EXPORT_SYMBOL(xp_dma_sync_for_cpu);
> +
> +void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
> + size_t size)
> +{
> + if (pool->cheap_dma)
> + return;
> +
> + dma_sync_single_range_for_device(pool->dev, dma, 0,
> + size, DMA_BIDIRECTIONAL);
> +}
> +EXPORT_SYMBOL(xp_dma_sync_for_device);
> diff --git a/net/xdp/xsk_diag.c b/net/xdp/xsk_diag.c
> index f59791ba43a0..0163b26aaf63 100644
> --- a/net/xdp/xsk_diag.c
> +++ b/net/xdp/xsk_diag.c
> @@ -56,7 +56,7 @@ static int xsk_diag_put_umem(const struct xdp_sock *xs, struct sk_buff *nlskb)
> du.id = umem->id;
> du.size = umem->size;
> du.num_pages = umem->npgs;
> - du.chunk_size = umem->chunk_size_nohr + umem->headroom;
> + du.chunk_size = umem->chunk_size;
> du.headroom = umem->headroom;
> du.ifindex = umem->dev ? umem->dev->ifindex : 0;
> du.queue_id = umem->queue_id;
> diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
> index a322a7dac58c..9151aef7dbca 100644
> --- a/net/xdp/xsk_queue.h
> +++ b/net/xdp/xsk_queue.h
> @@ -9,6 +9,7 @@
> #include <linux/types.h>
> #include <linux/if_xdp.h>
> #include <net/xdp_sock.h>
> +#include <net/xsk_buff_pool.h>
>
> #include "xsk.h"
>
> @@ -172,31 +173,45 @@ static inline bool xskq_cons_read_addr(struct xsk_queue *q, u64 *addr,
> return false;
> }
>
> -static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
> - struct xdp_desc *d,
> - struct xdp_umem *umem)
> +static inline bool xskq_cons_read_addr_aligned(struct xsk_queue *q, u64 *addr)
> {
> - if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG) {
> - if (!xskq_cons_is_valid_unaligned(q, d->addr, d->len, umem))
> - return false;
> + struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
>
> - if (d->len > umem->chunk_size_nohr || d->options) {
> - q->invalid_descs++;
> - return false;
> - }
> + while (q->cached_cons != q->cached_prod) {
> + u32 idx = q->cached_cons & q->ring_mask;
> +
> + *addr = ring->desc[idx];
> + if (xskq_cons_is_valid_addr(q, *addr))
> + return true;
>
> + q->cached_cons++;
> + }
> +
> + return false;
> +}
> +
> +static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr)
> +{
> + struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
> +
> + if (q->cached_cons != q->cached_prod) {
> + u32 idx = q->cached_cons & q->ring_mask;
> +
> + *addr = ring->desc[idx];
> return true;
> }
>
> - if (!xskq_cons_is_valid_addr(q, d->addr))
> - return false;
> + return false;
> +}
>
> - if (((d->addr + d->len) & q->chunk_mask) != (d->addr & q->chunk_mask) ||
> - d->options) {
> +static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
> + struct xdp_desc *d,
> + struct xdp_umem *umem)
> +{
> + if (!xp_validate_desc(umem->pool, d)) {
> q->invalid_descs++;
> return false;
> }
> -
> return true;
> }
>
> @@ -260,6 +275,20 @@ static inline bool xskq_cons_peek_addr(struct xsk_queue *q, u64 *addr,
> return xskq_cons_read_addr(q, addr, umem);
> }
>
> +static inline bool xskq_cons_peek_addr_aligned(struct xsk_queue *q, u64 *addr)
> +{
> + if (q->cached_prod == q->cached_cons)
> + xskq_cons_get_entries(q);
> + return xskq_cons_read_addr_aligned(q, addr);
> +}
> +
> +static inline bool xskq_cons_peek_addr_unchecked(struct xsk_queue *q, u64 *addr)
> +{
> + if (q->cached_prod == q->cached_cons)
> + xskq_cons_get_entries(q);
> + return xskq_cons_read_addr_unchecked(q, addr);
> +}
> +
> static inline bool xskq_cons_peek_desc(struct xsk_queue *q,
> struct xdp_desc *desc,
> struct xdp_umem *umem)
>
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [PATCH bpf-next 04/14] xsk: introduce AF_XDP buffer allocation API
2020-05-08 11:55 ` Maxim Mikityanskiy
@ 2020-05-08 12:32 ` Björn Töpel
0 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-08 12:32 UTC (permalink / raw)
To: Maxim Mikityanskiy, Björn Töpel
Cc: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher,
maciej.fijalkowski
On 2020-05-08 13:55, Maxim Mikityanskiy wrote:
> On 2020-05-07 13:42, Björn Töpel wrote:
>> From: Björn Töpel <bjorn.topel@intel.com>
>>
>> In order to simplify AF_XDP zero-copy enablement for NIC driver
>> developers, a new AF_XDP buffer allocation API is added. The
>> implementation is based on a single core (single producer/consumer)
>> buffer pool for the AF_XDP UMEM.
>>
>> A buffer is allocated using the xsk_buff_alloc() function, and
>> returned using xsk_buff_free(). If a buffer is disassociated with the
>> pool, e.g. when a buffer is passed to an AF_XDP socket, a buffer is
>> said to be released. Currently, the release function is only used by
>> the AF_XDP internals and not visible to the driver.
>>
>> Drivers using this API should register the XDP memory model with the
>> new MEM_TYPE_XSK_BUFF_POOL type.
>>
>> The API is defined in net/xdp_sock_drv.h.
>>
>> The buffer type is struct xdp_buff, and follows the lifetime of
>> regular xdp_buffs, i.e. the lifetime of an xdp_buff is restricted to
>> a NAPI context. In other words, the API is not replacing xdp_frames.
>>
>> In addition to introducing the API and implementations, the AF_XDP
>> core is migrated to use the new APIs.
>>
>> rfc->v1: Fixed build errors/warnings for m68k and riscv. (kbuild test
>> robot)
>> Added headroom/chunk size getter. (Maxim/Björn)
>>
>> Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
>> Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
>
> Björn's signoff should go first (as the original author).
>
Oh, I didn't know that. I'll fix in the next revision!
^ permalink raw reply [flat|nested] 24+ messages in thread
* [PATCH bpf-next 05/14] i40e: refactor rx_bi accesses
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (3 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 04/14] xsk: introduce AF_XDP buffer allocation API Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 06/14] i40e: separate kernel allocated rx_bi rings from AF_XDP rings Björn Töpel
` (8 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski, intel-wired-lan
From: Björn Töpel <bjorn.topel@intel.com>
As a first step to migrate i40e to the new MEM_TYPE_XSK_BUFF_POOL
APIs, code that accesses the rx_bi (SW/shadow ring) is refactored to
use an accessor function.
Cc: intel-wired-lan@lists.osuosl.org
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
drivers/net/ethernet/intel/i40e/i40e_txrx.c | 17 +++++++++++------
drivers/net/ethernet/intel/i40e/i40e_xsk.c | 18 ++++++++++++------
2 files changed, 23 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index b8496037ef7f..58daba8fabc8 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -1195,6 +1195,11 @@ static void i40e_update_itr(struct i40e_q_vector *q_vector,
rc->total_packets = 0;
}
+static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
+{
+ return &rx_ring->rx_bi[idx];
+}
+
/**
* i40e_reuse_rx_page - page flip buffer and store it back on the ring
* @rx_ring: rx descriptor ring to store buffers on
@@ -1208,7 +1213,7 @@ static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *new_buff;
u16 nta = rx_ring->next_to_alloc;
- new_buff = &rx_ring->rx_bi[nta];
+ new_buff = i40e_rx_bi(rx_ring, nta);
/* update, and store next to alloc */
nta++;
@@ -1272,7 +1277,7 @@ struct i40e_rx_buffer *i40e_clean_programming_status(
ntc = rx_ring->next_to_clean;
/* fetch, update, and store next to clean */
- rx_buffer = &rx_ring->rx_bi[ntc++];
+ rx_buffer = i40e_rx_bi(rx_ring, ntc++);
ntc = (ntc < rx_ring->count) ? ntc : 0;
rx_ring->next_to_clean = ntc;
@@ -1361,7 +1366,7 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
/* Free all the Rx ring sk_buffs */
for (i = 0; i < rx_ring->count; i++) {
- struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
+ struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i);
if (!rx_bi->page)
continue;
@@ -1576,7 +1581,7 @@ bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
return false;
rx_desc = I40E_RX_DESC(rx_ring, ntu);
- bi = &rx_ring->rx_bi[ntu];
+ bi = i40e_rx_bi(rx_ring, ntu);
do {
if (!i40e_alloc_mapped_page(rx_ring, bi))
@@ -1598,7 +1603,7 @@ bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
ntu++;
if (unlikely(ntu == rx_ring->count)) {
rx_desc = I40E_RX_DESC(rx_ring, 0);
- bi = rx_ring->rx_bi;
+ bi = i40e_rx_bi(rx_ring, 0);
ntu = 0;
}
@@ -1965,7 +1970,7 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
{
struct i40e_rx_buffer *rx_buffer;
- rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
+ rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
prefetchw(rx_buffer->page);
/* we are reusing so sync this buffer for CPU use */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index 452bba7bc4ff..8d29477bb0b6 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -9,6 +9,11 @@
#include "i40e_txrx_common.h"
#include "i40e_xsk.h"
+static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
+{
+ return &rx_ring->rx_bi[idx];
+}
+
/**
* i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev
* @vsi: Current VSI
@@ -321,7 +326,7 @@ __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
bool ok = true;
rx_desc = I40E_RX_DESC(rx_ring, ntu);
- bi = &rx_ring->rx_bi[ntu];
+ bi = i40e_rx_bi(rx_ring, ntu);
do {
if (!alloc(rx_ring, bi)) {
ok = false;
@@ -340,7 +345,7 @@ __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
if (unlikely(ntu == rx_ring->count)) {
rx_desc = I40E_RX_DESC(rx_ring, 0);
- bi = rx_ring->rx_bi;
+ bi = i40e_rx_bi(rx_ring, 0);
ntu = 0;
}
@@ -402,7 +407,7 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring,
{
struct i40e_rx_buffer *bi;
- bi = &rx_ring->rx_bi[rx_ring->next_to_clean];
+ bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
/* we are reusing so sync this buffer for CPU use */
dma_sync_single_range_for_cpu(rx_ring->dev,
@@ -424,7 +429,8 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring,
static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *old_bi)
{
- struct i40e_rx_buffer *new_bi = &rx_ring->rx_bi[rx_ring->next_to_alloc];
+ struct i40e_rx_buffer *new_bi = i40e_rx_bi(rx_ring,
+ rx_ring->next_to_alloc);
u16 nta = rx_ring->next_to_alloc;
/* update, and store next to alloc */
@@ -456,7 +462,7 @@ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
mask = rx_ring->xsk_umem->chunk_mask;
nta = rx_ring->next_to_alloc;
- bi = &rx_ring->rx_bi[nta];
+ bi = i40e_rx_bi(rx_ring, nta);
nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
@@ -824,7 +830,7 @@ void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)
u16 i;
for (i = 0; i < rx_ring->count; i++) {
- struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
+ struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i);
if (!rx_bi->addr)
continue;
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 06/14] i40e: separate kernel allocated rx_bi rings from AF_XDP rings
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (4 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 05/14] i40e: refactor rx_bi accesses Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 07/14] i40e, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL Björn Töpel
` (7 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski, intel-wired-lan
From: Björn Töpel <bjorn.topel@intel.com>
Continuing the path to support MEM_TYPE_XSK_BUFF_POOL, the AF_XDP
zero-copy/sk_buff rx_bi rings are now separate. Functions to properly
allocate the different rings are added as well.
Cc: intel-wired-lan@lists.osuosl.org
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
drivers/net/ethernet/intel/i40e/i40e_main.c | 7 ++
drivers/net/ethernet/intel/i40e/i40e_txrx.c | 119 +++++++-----------
drivers/net/ethernet/intel/i40e/i40e_txrx.h | 22 ++--
.../ethernet/intel/i40e/i40e_txrx_common.h | 40 +++++-
drivers/net/ethernet/intel/i40e/i40e_type.h | 5 +-
drivers/net/ethernet/intel/i40e/i40e_xsk.c | 74 ++++++-----
drivers/net/ethernet/intel/i40e/i40e_xsk.h | 2 +
7 files changed, 142 insertions(+), 127 deletions(-)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index d6b2db4f2c65..3e1695bb8262 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -3260,8 +3260,12 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
if (ring->vsi->type == I40E_VSI_MAIN)
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
+ kfree(ring->rx_bi);
ring->xsk_umem = i40e_xsk_umem(ring);
if (ring->xsk_umem) {
+ ret = i40e_alloc_rx_bi_zc(ring);
+ if (ret)
+ return ret;
ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
XDP_PACKET_HEADROOM;
/* For AF_XDP ZC, we disallow packets to span on
@@ -3280,6 +3284,9 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
ring->queue_index);
} else {
+ ret = i40e_alloc_rx_bi(ring);
+ if (ret)
+ return ret;
ring->rx_buf_len = vsi->rx_buf_len;
if (ring->vsi->type == I40E_VSI_MAIN) {
ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index 58daba8fabc8..f063df623443 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -521,28 +521,29 @@ int i40e_add_del_fdir(struct i40e_vsi *vsi,
/**
* i40e_fd_handle_status - check the Programming Status for FD
* @rx_ring: the Rx ring for this descriptor
- * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
+ * @qword0_raw: qword0
+ * @qword1: qword1 after le_to_cpu
* @prog_id: the id originally used for programming
*
* This is used to verify if the FD programming or invalidation
* requested by SW to the HW is successful or not and take actions accordingly.
**/
-void i40e_fd_handle_status(struct i40e_ring *rx_ring,
- union i40e_rx_desc *rx_desc, u8 prog_id)
+void i40e_fd_handle_status(struct i40e_ring *rx_ring, u64 qword0_raw,
+ u64 qword1, u8 prog_id)
{
struct i40e_pf *pf = rx_ring->vsi->back;
struct pci_dev *pdev = pf->pdev;
+ struct i40e_32b_rx_wb_qw0 *qw0;
u32 fcnt_prog, fcnt_avail;
u32 error;
- u64 qw;
- qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
- error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
+ qw0 = (struct i40e_32b_rx_wb_qw0 *)&qword0_raw;
+ error = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
- pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id);
- if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) ||
+ pf->fd_inv = le32_to_cpu(qw0->hi_dword.fd_id);
+ if (qw0->hi_dword.fd_id != 0 ||
(I40E_DEBUG_FD & pf->hw.debug_mask))
dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
pf->fd_inv);
@@ -560,7 +561,7 @@ void i40e_fd_handle_status(struct i40e_ring *rx_ring,
/* store the current atr filter count */
pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf);
- if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) &&
+ if (qw0->hi_dword.fd_id == 0 &&
test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) {
/* These set_bit() calls aren't atomic with the
* test_bit() here, but worse case we potentially
@@ -589,7 +590,7 @@ void i40e_fd_handle_status(struct i40e_ring *rx_ring,
} else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
if (I40E_DEBUG_FD & pf->hw.debug_mask)
dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n",
- rx_desc->wb.qword0.hi_dword.fd_id);
+ qw0->hi_dword.fd_id);
}
}
@@ -1232,29 +1233,10 @@ static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
}
/**
- * i40e_rx_is_programming_status - check for programming status descriptor
- * @qw: qword representing status_error_len in CPU ordering
- *
- * The value of in the descriptor length field indicate if this
- * is a programming status descriptor for flow director or FCoE
- * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise
- * it is a packet descriptor.
- **/
-static inline bool i40e_rx_is_programming_status(u64 qw)
-{
- /* The Rx filter programming status and SPH bit occupy the same
- * spot in the descriptor. Since we don't support packet split we
- * can just reuse the bit as an indication that this is a
- * programming status descriptor.
- */
- return qw & I40E_RXD_QW1_LENGTH_SPH_MASK;
-}
-
-/**
- * i40e_clean_programming_status - try clean the programming status descriptor
+ * i40e_clean_programming_status - clean the programming status descriptor
* @rx_ring: the rx ring that has this descriptor
- * @rx_desc: the rx descriptor written back by HW
- * @qw: qword representing status_error_len in CPU ordering
+ * @qword0: qword0
+ * @qword1: qword1 representing status_error_len in CPU ordering
*
* Flow director should handle FD_FILTER_STATUS to check its filter programming
* status being successful or not and take actions accordingly. FCoE should
@@ -1262,34 +1244,16 @@ static inline bool i40e_rx_is_programming_status(u64 qw)
*
* Returns an i40e_rx_buffer to reuse if the cleanup occurred, otherwise NULL.
**/
-struct i40e_rx_buffer *i40e_clean_programming_status(
- struct i40e_ring *rx_ring,
- union i40e_rx_desc *rx_desc,
- u64 qw)
+void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw,
+ u64 qword1)
{
- struct i40e_rx_buffer *rx_buffer;
- u32 ntc;
u8 id;
- if (!i40e_rx_is_programming_status(qw))
- return NULL;
-
- ntc = rx_ring->next_to_clean;
-
- /* fetch, update, and store next to clean */
- rx_buffer = i40e_rx_bi(rx_ring, ntc++);
- ntc = (ntc < rx_ring->count) ? ntc : 0;
- rx_ring->next_to_clean = ntc;
-
- prefetch(I40E_RX_DESC(rx_ring, ntc));
-
- id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
+ id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
- i40e_fd_handle_status(rx_ring, rx_desc, id);
-
- return rx_buffer;
+ i40e_fd_handle_status(rx_ring, qword0_raw, qword1, id);
}
/**
@@ -1341,13 +1305,25 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
return -ENOMEM;
}
+int i40e_alloc_rx_bi(struct i40e_ring *rx_ring)
+{
+ unsigned long sz = sizeof(*rx_ring->rx_bi) * rx_ring->count;
+
+ rx_ring->rx_bi = kzalloc(sz, GFP_KERNEL);
+ return rx_ring->rx_bi ? 0 : -ENOMEM;
+}
+
+static void i40e_clear_rx_bi(struct i40e_ring *rx_ring)
+{
+ memset(rx_ring->rx_bi, 0, sizeof(*rx_ring->rx_bi) * rx_ring->count);
+}
+
/**
* i40e_clean_rx_ring - Free Rx buffers
* @rx_ring: ring to be cleaned
**/
void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
{
- unsigned long bi_size;
u16 i;
/* ring already cleared, nothing to do */
@@ -1393,8 +1369,10 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
}
skip_free:
- bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
- memset(rx_ring->rx_bi, 0, bi_size);
+ if (rx_ring->xsk_umem)
+ i40e_clear_rx_bi_zc(rx_ring);
+ else
+ i40e_clear_rx_bi(rx_ring);
/* Zero out the descriptor ring */
memset(rx_ring->desc, 0, rx_ring->size);
@@ -1435,15 +1413,7 @@ void i40e_free_rx_resources(struct i40e_ring *rx_ring)
int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
{
struct device *dev = rx_ring->dev;
- int err = -ENOMEM;
- int bi_size;
-
- /* warn if we are about to overwrite the pointer */
- WARN_ON(rx_ring->rx_bi);
- bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
- rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
- if (!rx_ring->rx_bi)
- goto err;
+ int err;
u64_stats_init(&rx_ring->syncp);
@@ -1456,7 +1426,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
if (!rx_ring->desc) {
dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
rx_ring->size);
- goto err;
+ return -ENOMEM;
}
rx_ring->next_to_alloc = 0;
@@ -1468,16 +1438,12 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
rx_ring->queue_index);
if (err < 0)
- goto err;
+ return err;
}
rx_ring->xdp_prog = rx_ring->vsi->xdp_prog;
return 0;
-err:
- kfree(rx_ring->rx_bi);
- rx_ring->rx_bi = NULL;
- return err;
}
/**
@@ -2370,9 +2336,12 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
*/
dma_rmb();
- rx_buffer = i40e_clean_programming_status(rx_ring, rx_desc,
- qword);
- if (unlikely(rx_buffer)) {
+ if (i40e_rx_is_programming_status(qword)) {
+ i40e_clean_programming_status(rx_ring,
+ rx_desc->raw.qword[0],
+ qword);
+ rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
+ i40e_inc_ntc(rx_ring);
i40e_reuse_rx_page(rx_ring, rx_buffer);
cleaned_count++;
continue;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index 36d37f31a287..d343498e8de5 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -296,17 +296,15 @@ struct i40e_tx_buffer {
struct i40e_rx_buffer {
dma_addr_t dma;
- union {
- struct {
- struct page *page;
- __u32 page_offset;
- __u16 pagecnt_bias;
- };
- struct {
- void *addr;
- u64 handle;
- };
- };
+ struct page *page;
+ __u32 page_offset;
+ __u16 pagecnt_bias;
+};
+
+struct i40e_rx_buffer_zc {
+ dma_addr_t dma;
+ void *addr;
+ u64 handle;
};
struct i40e_queue_stats {
@@ -358,6 +356,7 @@ struct i40e_ring {
union {
struct i40e_tx_buffer *tx_bi;
struct i40e_rx_buffer *rx_bi;
+ struct i40e_rx_buffer_zc *rx_bi_zc;
};
DECLARE_BITMAP(state, __I40E_RING_STATE_NBITS);
u16 queue_index; /* Queue number of ring */
@@ -495,6 +494,7 @@ int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
bool __i40e_chk_linearize(struct sk_buff *skb);
int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
u32 flags);
+int i40e_alloc_rx_bi(struct i40e_ring *rx_ring);
/**
* i40e_get_head - Retrieve head from head writeback
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h b/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h
index 8af0e99c6c0d..667c4dc4b39f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx_common.h
@@ -4,13 +4,9 @@
#ifndef I40E_TXRX_COMMON_
#define I40E_TXRX_COMMON_
-void i40e_fd_handle_status(struct i40e_ring *rx_ring,
- union i40e_rx_desc *rx_desc, u8 prog_id);
int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring);
-struct i40e_rx_buffer *i40e_clean_programming_status(
- struct i40e_ring *rx_ring,
- union i40e_rx_desc *rx_desc,
- u64 qw);
+void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw,
+ u64 qword1);
void i40e_process_skb_fields(struct i40e_ring *rx_ring,
union i40e_rx_desc *rx_desc, struct sk_buff *skb);
void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring);
@@ -84,6 +80,38 @@ static inline void i40e_arm_wb(struct i40e_ring *tx_ring,
}
}
+/**
+ * i40e_rx_is_programming_status - check for programming status descriptor
+ * @qword1: qword1 representing status_error_len in CPU ordering
+ *
+ * The value of in the descriptor length field indicate if this
+ * is a programming status descriptor for flow director or FCoE
+ * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise
+ * it is a packet descriptor.
+ **/
+static inline bool i40e_rx_is_programming_status(u64 qword1)
+{
+ /* The Rx filter programming status and SPH bit occupy the same
+ * spot in the descriptor. Since we don't support packet split we
+ * can just reuse the bit as an indication that this is a
+ * programming status descriptor.
+ */
+ return qword1 & I40E_RXD_QW1_LENGTH_SPH_MASK;
+}
+
+/**
+ * i40e_inc_ntc: Advance the next_to_clean index
+ * @rx_ring: Rx ring
+ **/
+static inline void i40e_inc_ntc(struct i40e_ring *rx_ring)
+{
+ u32 ntc = rx_ring->next_to_clean + 1;
+
+ ntc = (ntc < rx_ring->count) ? ntc : 0;
+ rx_ring->next_to_clean = ntc;
+ prefetch(I40E_RX_DESC(rx_ring, ntc));
+}
+
void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring);
void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring);
bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h
index 6ea2867ff60f..63e098f7cb63 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_type.h
@@ -689,7 +689,7 @@ union i40e_32byte_rx_desc {
__le64 rsvd2;
} read;
struct {
- struct {
+ struct i40e_32b_rx_wb_qw0 {
struct {
union {
__le16 mirroring_status;
@@ -727,6 +727,9 @@ union i40e_32byte_rx_desc {
} hi_dword;
} qword3;
} wb; /* writeback */
+ struct {
+ u64 qword[4];
+ } raw;
};
enum i40e_rx_desc_status_bits {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index 8d29477bb0b6..4fce057f1eec 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -9,9 +9,23 @@
#include "i40e_txrx_common.h"
#include "i40e_xsk.h"
-static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
+int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring)
{
- return &rx_ring->rx_bi[idx];
+ unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count;
+
+ rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL);
+ return rx_ring->rx_bi_zc ? 0 : -ENOMEM;
+}
+
+void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring)
+{
+ memset(rx_ring->rx_bi_zc, 0,
+ sizeof(*rx_ring->rx_bi_zc) * rx_ring->count);
+}
+
+static struct i40e_rx_buffer_zc *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
+{
+ return &rx_ring->rx_bi_zc[idx];
}
/**
@@ -238,7 +252,7 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
}
/**
- * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer
+ * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer_zc
* @rx_ring: Rx ring
* @bi: Rx buffer to populate
*
@@ -248,7 +262,7 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
* Returns true for a successful allocation, false otherwise
**/
static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer *bi)
+ struct i40e_rx_buffer_zc *bi)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
void *addr = bi->addr;
@@ -279,7 +293,7 @@ static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring,
}
/**
- * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer
+ * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer_zc
* @rx_ring: Rx ring
* @bi: Rx buffer to populate
*
@@ -289,7 +303,7 @@ static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring,
* Returns true for a successful allocation, false otherwise
**/
static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer *bi)
+ struct i40e_rx_buffer_zc *bi)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
u64 handle, hr;
@@ -318,11 +332,11 @@ static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring,
static __always_inline bool
__i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
bool alloc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer *bi))
+ struct i40e_rx_buffer_zc *bi))
{
u16 ntu = rx_ring->next_to_use;
union i40e_rx_desc *rx_desc;
- struct i40e_rx_buffer *bi;
+ struct i40e_rx_buffer_zc *bi;
bool ok = true;
rx_desc = I40E_RX_DESC(rx_ring, ntu);
@@ -402,10 +416,11 @@ static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count)
*
* Returns the received Rx buffer
**/
-static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring,
- const unsigned int size)
+static struct i40e_rx_buffer_zc *i40e_get_rx_buffer_zc(
+ struct i40e_ring *rx_ring,
+ const unsigned int size)
{
- struct i40e_rx_buffer *bi;
+ struct i40e_rx_buffer_zc *bi;
bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
@@ -427,10 +442,10 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring,
* recycle queue (next_to_alloc).
**/
static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer *old_bi)
+ struct i40e_rx_buffer_zc *old_bi)
{
- struct i40e_rx_buffer *new_bi = i40e_rx_bi(rx_ring,
- rx_ring->next_to_alloc);
+ struct i40e_rx_buffer_zc *new_bi = i40e_rx_bi(rx_ring,
+ rx_ring->next_to_alloc);
u16 nta = rx_ring->next_to_alloc;
/* update, and store next to alloc */
@@ -452,7 +467,7 @@ static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring,
**/
void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
{
- struct i40e_rx_buffer *bi;
+ struct i40e_rx_buffer_zc *bi;
struct i40e_ring *rx_ring;
u64 hr, mask;
u16 nta;
@@ -490,7 +505,7 @@ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
* Returns the skb, or NULL on failure.
**/
static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer *bi,
+ struct i40e_rx_buffer_zc *bi,
struct xdp_buff *xdp)
{
unsigned int metasize = xdp->data - xdp->data_meta;
@@ -513,19 +528,6 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
return skb;
}
-/**
- * i40e_inc_ntc: Advance the next_to_clean index
- * @rx_ring: Rx ring
- **/
-static void i40e_inc_ntc(struct i40e_ring *rx_ring)
-{
- u32 ntc = rx_ring->next_to_clean + 1;
-
- ntc = (ntc < rx_ring->count) ? ntc : 0;
- rx_ring->next_to_clean = ntc;
- prefetch(I40E_RX_DESC(rx_ring, ntc));
-}
-
/**
* i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring
* @rx_ring: Rx ring
@@ -545,7 +547,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
xdp.rxq = &rx_ring->xdp_rxq;
while (likely(total_rx_packets < (unsigned int)budget)) {
- struct i40e_rx_buffer *bi;
+ struct i40e_rx_buffer_zc *bi;
union i40e_rx_desc *rx_desc;
unsigned int size;
u64 qword;
@@ -566,14 +568,18 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
*/
dma_rmb();
- bi = i40e_clean_programming_status(rx_ring, rx_desc,
- qword);
- if (unlikely(bi)) {
+ if (i40e_rx_is_programming_status(qword)) {
+ i40e_clean_programming_status(rx_ring,
+ rx_desc->raw.qword[0],
+ qword);
+ bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
+ i40e_inc_ntc(rx_ring);
i40e_reuse_rx_buffer_zc(rx_ring, bi);
cleaned_count++;
continue;
}
+ bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
if (!size)
@@ -830,7 +836,7 @@ void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)
u16 i;
for (i = 0; i < rx_ring->count; i++) {
- struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i);
+ struct i40e_rx_buffer_zc *rx_bi = i40e_rx_bi(rx_ring, i);
if (!rx_bi->addr)
continue;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.h b/drivers/net/ethernet/intel/i40e/i40e_xsk.h
index 9ed59c14eb55..f5e292c218ee 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.h
@@ -19,5 +19,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget);
bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
struct i40e_ring *tx_ring, int napi_budget);
int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
+int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring);
+void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring);
#endif /* _I40E_XSK_H_ */
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 07/14] i40e, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (5 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 06/14] i40e: separate kernel allocated rx_bi rings from AF_XDP rings Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 08/14] ice, " Björn Töpel
` (6 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski, intel-wired-lan
From: Björn Töpel <bjorn.topel@intel.com>
Remove MEM_TYPE_ZERO_COPY in favor of the new MEM_TYPE_XSK_BUFF_POOL
APIs. The AF_XDP zero-copy rx_bi ring is now simply a struct xdp_buff
pointer.
Cc: intel-wired-lan@lists.osuosl.org
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
drivers/net/ethernet/intel/i40e/i40e_main.c | 19 +-
drivers/net/ethernet/intel/i40e/i40e_txrx.h | 9 +-
drivers/net/ethernet/intel/i40e/i40e_xsk.c | 350 ++------------------
drivers/net/ethernet/intel/i40e/i40e_xsk.h | 1 -
4 files changed, 47 insertions(+), 332 deletions(-)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 3e1695bb8262..ea7395b391e5 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -3266,21 +3266,19 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
ret = i40e_alloc_rx_bi_zc(ring);
if (ret)
return ret;
- ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
- XDP_PACKET_HEADROOM;
+ ring->rx_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem);
/* For AF_XDP ZC, we disallow packets to span on
* multiple buffers, thus letting us skip that
* handling in the fast-path.
*/
chain_len = 1;
- ring->zca.free = i40e_zca_free;
ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
- MEM_TYPE_ZERO_COPY,
- &ring->zca);
+ MEM_TYPE_XSK_BUFF_POOL,
+ NULL);
if (ret)
return ret;
dev_info(&vsi->back->pdev->dev,
- "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
+ "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
ring->queue_index);
} else {
@@ -3351,9 +3349,12 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
writel(0, ring->tail);
- ok = ring->xsk_umem ?
- i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) :
- !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
+ if (ring->xsk_umem) {
+ xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq);
+ ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
+ } else {
+ ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
+ }
if (!ok) {
/* Log this in case the user has forgotten to give the kernel
* any buffers, even later in the application.
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index d343498e8de5..5c255977fd58 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -301,12 +301,6 @@ struct i40e_rx_buffer {
__u16 pagecnt_bias;
};
-struct i40e_rx_buffer_zc {
- dma_addr_t dma;
- void *addr;
- u64 handle;
-};
-
struct i40e_queue_stats {
u64 packets;
u64 bytes;
@@ -356,7 +350,7 @@ struct i40e_ring {
union {
struct i40e_tx_buffer *tx_bi;
struct i40e_rx_buffer *rx_bi;
- struct i40e_rx_buffer_zc *rx_bi_zc;
+ struct xdp_buff **rx_bi_zc;
};
DECLARE_BITMAP(state, __I40E_RING_STATE_NBITS);
u16 queue_index; /* Queue number of ring */
@@ -418,7 +412,6 @@ struct i40e_ring {
struct i40e_channel *ch;
struct xdp_rxq_info xdp_rxq;
struct xdp_umem *xsk_umem;
- struct zero_copy_allocator zca; /* ZC allocator anchor */
} ____cacheline_internodealigned_in_smp;
static inline bool ring_uses_build_skb(struct i40e_ring *ring)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index 4fce057f1eec..460f5052e1db 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -23,68 +23,11 @@ void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring)
sizeof(*rx_ring->rx_bi_zc) * rx_ring->count);
}
-static struct i40e_rx_buffer_zc *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
+static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
{
return &rx_ring->rx_bi_zc[idx];
}
-/**
- * i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev
- * @vsi: Current VSI
- * @umem: UMEM to DMA map
- *
- * Returns 0 on success, <0 on failure
- **/
-static int i40e_xsk_umem_dma_map(struct i40e_vsi *vsi, struct xdp_umem *umem)
-{
- struct i40e_pf *pf = vsi->back;
- struct device *dev;
- unsigned int i, j;
- dma_addr_t dma;
-
- dev = &pf->pdev->dev;
- for (i = 0; i < umem->npgs; i++) {
- dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
- if (dma_mapping_error(dev, dma))
- goto out_unmap;
-
- umem->pages[i].dma = dma;
- }
-
- return 0;
-
-out_unmap:
- for (j = 0; j < i; j++) {
- dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
- umem->pages[i].dma = 0;
- }
-
- return -1;
-}
-
-/**
- * i40e_xsk_umem_dma_unmap - DMA unmaps all UMEM memory for the netdev
- * @vsi: Current VSI
- * @umem: UMEM to DMA map
- **/
-static void i40e_xsk_umem_dma_unmap(struct i40e_vsi *vsi, struct xdp_umem *umem)
-{
- struct i40e_pf *pf = vsi->back;
- struct device *dev;
- unsigned int i;
-
- dev = &pf->pdev->dev;
-
- for (i = 0; i < umem->npgs; i++) {
- dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
-
- umem->pages[i].dma = 0;
- }
-}
-
/**
* i40e_xsk_umem_enable - Enable/associate a UMEM to a certain ring/qid
* @vsi: Current VSI
@@ -97,7 +40,6 @@ static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem,
u16 qid)
{
struct net_device *netdev = vsi->netdev;
- struct xdp_umem_fq_reuse *reuseq;
bool if_running;
int err;
@@ -111,13 +53,7 @@ static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem,
qid >= netdev->real_num_tx_queues)
return -EINVAL;
- reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
- if (!reuseq)
- return -ENOMEM;
-
- xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
-
- err = i40e_xsk_umem_dma_map(vsi, umem);
+ err = xsk_buff_dma_map(umem, &vsi->back->pdev->dev, I40E_RX_DMA_ATTR);
if (err)
return err;
@@ -170,7 +106,7 @@ static int i40e_xsk_umem_disable(struct i40e_vsi *vsi, u16 qid)
}
clear_bit(qid, vsi->af_xdp_zc_qps);
- i40e_xsk_umem_dma_unmap(vsi, umem);
+ xsk_buff_dma_unmap(umem, I40E_RX_DMA_ATTR);
if (if_running) {
err = i40e_queue_pair_enable(vsi, qid);
@@ -209,11 +145,9 @@ int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,
**/
static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
{
- struct xdp_umem *umem = rx_ring->xsk_umem;
int err, result = I40E_XDP_PASS;
struct i40e_ring *xdp_ring;
struct bpf_prog *xdp_prog;
- u64 offset;
u32 act;
rcu_read_lock();
@@ -222,9 +156,6 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
*/
xdp_prog = READ_ONCE(rx_ring->xdp_prog);
act = bpf_prog_run_xdp(xdp_prog, xdp);
- offset = xdp->data - xdp->data_hard_start;
-
- xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset);
switch (act) {
case XDP_PASS:
@@ -251,107 +182,26 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
return result;
}
-/**
- * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer_zc
- * @rx_ring: Rx ring
- * @bi: Rx buffer to populate
- *
- * This function allocates an Rx buffer. The buffer can come from fill
- * queue, or via the recycle queue (next_to_alloc).
- *
- * Returns true for a successful allocation, false otherwise
- **/
-static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer_zc *bi)
-{
- struct xdp_umem *umem = rx_ring->xsk_umem;
- void *addr = bi->addr;
- u64 handle, hr;
-
- if (addr) {
- rx_ring->rx_stats.page_reuse_count++;
- return true;
- }
-
- if (!xsk_umem_peek_addr(umem, &handle)) {
- rx_ring->rx_stats.alloc_page_failed++;
- return false;
- }
-
- hr = umem->headroom + XDP_PACKET_HEADROOM;
-
- bi->dma = xdp_umem_get_dma(umem, handle);
- bi->dma += hr;
-
- bi->addr = xdp_umem_get_data(umem, handle);
- bi->addr += hr;
-
- bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
-
- xsk_umem_release_addr(umem);
- return true;
-}
-
-/**
- * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer_zc
- * @rx_ring: Rx ring
- * @bi: Rx buffer to populate
- *
- * This function allocates an Rx buffer. The buffer can come from fill
- * queue, or via the reuse queue.
- *
- * Returns true for a successful allocation, false otherwise
- **/
-static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer_zc *bi)
-{
- struct xdp_umem *umem = rx_ring->xsk_umem;
- u64 handle, hr;
-
- if (!xsk_umem_peek_addr_rq(umem, &handle)) {
- rx_ring->rx_stats.alloc_page_failed++;
- return false;
- }
-
- handle &= rx_ring->xsk_umem->chunk_mask;
-
- hr = umem->headroom + XDP_PACKET_HEADROOM;
-
- bi->dma = xdp_umem_get_dma(umem, handle);
- bi->dma += hr;
-
- bi->addr = xdp_umem_get_data(umem, handle);
- bi->addr += hr;
-
- bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
-
- xsk_umem_release_addr_rq(umem);
- return true;
-}
-
-static __always_inline bool
-__i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
- bool alloc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer_zc *bi))
+bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
{
u16 ntu = rx_ring->next_to_use;
union i40e_rx_desc *rx_desc;
- struct i40e_rx_buffer_zc *bi;
+ struct xdp_buff **bi, *xdp;
+ dma_addr_t dma;
bool ok = true;
rx_desc = I40E_RX_DESC(rx_ring, ntu);
bi = i40e_rx_bi(rx_ring, ntu);
do {
- if (!alloc(rx_ring, bi)) {
+ xdp = xsk_buff_alloc(rx_ring->xsk_umem);
+ if (!xdp) {
ok = false;
goto no_buffers;
}
-
- dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 0,
- rx_ring->rx_buf_len,
- DMA_BIDIRECTIONAL);
-
- rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
+ *bi = xdp;
+ dma = xsk_buff_xdp_get_dma(xdp);
+ rx_desc->read.pkt_addr = cpu_to_le64(dma);
+ rx_desc->read.hdr_addr = 0;
rx_desc++;
bi++;
@@ -363,7 +213,6 @@ __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
ntu = 0;
}
- rx_desc->wb.qword1.status_error_len = 0;
count--;
} while (count);
@@ -374,126 +223,6 @@ __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
return ok;
}
-/**
- * i40e_alloc_rx_buffers_zc - Allocates a number of Rx buffers
- * @rx_ring: Rx ring
- * @count: The number of buffers to allocate
- *
- * This function allocates a number of Rx buffers from the reuse queue
- * or fill ring and places them on the Rx ring.
- *
- * Returns true for a successful allocation, false otherwise
- **/
-bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
-{
- return __i40e_alloc_rx_buffers_zc(rx_ring, count,
- i40e_alloc_buffer_slow_zc);
-}
-
-/**
- * i40e_alloc_rx_buffers_fast_zc - Allocates a number of Rx buffers
- * @rx_ring: Rx ring
- * @count: The number of buffers to allocate
- *
- * This function allocates a number of Rx buffers from the fill ring
- * or the internal recycle mechanism and places them on the Rx ring.
- *
- * Returns true for a successful allocation, false otherwise
- **/
-static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count)
-{
- return __i40e_alloc_rx_buffers_zc(rx_ring, count,
- i40e_alloc_buffer_zc);
-}
-
-/**
- * i40e_get_rx_buffer_zc - Return the current Rx buffer
- * @rx_ring: Rx ring
- * @size: The size of the rx buffer (read from descriptor)
- *
- * This function returns the current, received Rx buffer, and also
- * does DMA synchronization. the Rx ring.
- *
- * Returns the received Rx buffer
- **/
-static struct i40e_rx_buffer_zc *i40e_get_rx_buffer_zc(
- struct i40e_ring *rx_ring,
- const unsigned int size)
-{
- struct i40e_rx_buffer_zc *bi;
-
- bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
-
- /* we are reusing so sync this buffer for CPU use */
- dma_sync_single_range_for_cpu(rx_ring->dev,
- bi->dma, 0,
- size,
- DMA_BIDIRECTIONAL);
-
- return bi;
-}
-
-/**
- * i40e_reuse_rx_buffer_zc - Recycle an Rx buffer
- * @rx_ring: Rx ring
- * @old_bi: The Rx buffer to recycle
- *
- * This function recycles a finished Rx buffer, and places it on the
- * recycle queue (next_to_alloc).
- **/
-static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer_zc *old_bi)
-{
- struct i40e_rx_buffer_zc *new_bi = i40e_rx_bi(rx_ring,
- rx_ring->next_to_alloc);
- u16 nta = rx_ring->next_to_alloc;
-
- /* update, and store next to alloc */
- nta++;
- rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
-
- /* transfer page from old buffer to new buffer */
- new_bi->dma = old_bi->dma;
- new_bi->addr = old_bi->addr;
- new_bi->handle = old_bi->handle;
-
- old_bi->addr = NULL;
-}
-
-/**
- * i40e_zca_free - Free callback for MEM_TYPE_ZERO_COPY allocations
- * @alloc: Zero-copy allocator
- * @handle: Buffer handle
- **/
-void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
-{
- struct i40e_rx_buffer_zc *bi;
- struct i40e_ring *rx_ring;
- u64 hr, mask;
- u16 nta;
-
- rx_ring = container_of(alloc, struct i40e_ring, zca);
- hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
- mask = rx_ring->xsk_umem->chunk_mask;
-
- nta = rx_ring->next_to_alloc;
- bi = i40e_rx_bi(rx_ring, nta);
-
- nta++;
- rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
-
- handle &= mask;
-
- bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle);
- bi->dma += hr;
-
- bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle);
- bi->addr += hr;
-
- bi->handle = xsk_umem_adjust_offset(rx_ring->xsk_umem, (u64)handle,
- rx_ring->xsk_umem->headroom);
-}
-
/**
* i40e_construct_skb_zc - Create skbufff from zero-copy Rx buffer
* @rx_ring: Rx ring
@@ -505,7 +234,6 @@ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
* Returns the skb, or NULL on failure.
**/
static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
- struct i40e_rx_buffer_zc *bi,
struct xdp_buff *xdp)
{
unsigned int metasize = xdp->data - xdp->data_meta;
@@ -524,7 +252,7 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
if (metasize)
skb_metadata_set(skb, metasize);
- i40e_reuse_rx_buffer_zc(rx_ring, bi);
+ xsk_buff_free(xdp);
return skb;
}
@@ -542,20 +270,17 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
unsigned int xdp_res, xdp_xmit = 0;
bool failure = false;
struct sk_buff *skb;
- struct xdp_buff xdp;
-
- xdp.rxq = &rx_ring->xdp_rxq;
while (likely(total_rx_packets < (unsigned int)budget)) {
- struct i40e_rx_buffer_zc *bi;
union i40e_rx_desc *rx_desc;
+ struct xdp_buff **bi;
unsigned int size;
u64 qword;
if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
failure = failure ||
- !i40e_alloc_rx_buffers_fast_zc(rx_ring,
- cleaned_count);
+ !i40e_alloc_rx_buffers_zc(rx_ring,
+ cleaned_count);
cleaned_count = 0;
}
@@ -573,9 +298,10 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
rx_desc->raw.qword[0],
qword);
bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
- i40e_inc_ntc(rx_ring);
- i40e_reuse_rx_buffer_zc(rx_ring, bi);
+ xsk_buff_free(*bi);
+ *bi = NULL;
cleaned_count++;
+ i40e_inc_ntc(rx_ring);
continue;
}
@@ -585,22 +311,18 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
if (!size)
break;
- bi = i40e_get_rx_buffer_zc(rx_ring, size);
- xdp.data = bi->addr;
- xdp.data_meta = xdp.data;
- xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
- xdp.data_end = xdp.data + size;
- xdp.handle = bi->handle;
+ bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
+ (*bi)->data_end = (*bi)->data + size;
+ xsk_buff_dma_sync_for_cpu(*bi);
- xdp_res = i40e_run_xdp_zc(rx_ring, &xdp);
+ xdp_res = i40e_run_xdp_zc(rx_ring, *bi);
if (xdp_res) {
- if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) {
+ if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR))
xdp_xmit |= xdp_res;
- bi->addr = NULL;
- } else {
- i40e_reuse_rx_buffer_zc(rx_ring, bi);
- }
+ else
+ xsk_buff_free(*bi);
+ *bi = NULL;
total_rx_bytes += size;
total_rx_packets++;
@@ -616,7 +338,8 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
* BIT(I40E_RXD_QW1_ERROR_SHIFT). This is due to that
* SBP is *not* set in PRT_SBPVSI (default not set).
*/
- skb = i40e_construct_skb_zc(rx_ring, bi, &xdp);
+ skb = i40e_construct_skb_zc(rx_ring, *bi);
+ *bi = NULL;
if (!skb) {
rx_ring->rx_stats.alloc_buff_failed++;
break;
@@ -674,10 +397,9 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
break;
- dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
-
- dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
- DMA_BIDIRECTIONAL);
+ dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr);
+ xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma,
+ desc.len);
tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use];
tx_bi->bytecount = desc.len;
@@ -836,13 +558,13 @@ void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)
u16 i;
for (i = 0; i < rx_ring->count; i++) {
- struct i40e_rx_buffer_zc *rx_bi = i40e_rx_bi(rx_ring, i);
+ struct xdp_buff *rx_bi = *i40e_rx_bi(rx_ring, i);
- if (!rx_bi->addr)
+ if (!rx_bi)
continue;
- xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_bi->handle);
- rx_bi->addr = NULL;
+ xsk_buff_free(rx_bi);
+ rx_bi = NULL;
}
}
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.h b/drivers/net/ethernet/intel/i40e/i40e_xsk.h
index f5e292c218ee..ea919a7d60ec 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.h
@@ -12,7 +12,6 @@ int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair);
int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair);
int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,
u16 qid);
-void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);
bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 cleaned_count);
int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget);
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 08/14] ice, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (6 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 07/14] i40e, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 09/14] ixgbe, " Björn Töpel
` (5 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski, intel-wired-lan
From: Björn Töpel <bjorn.topel@intel.com>
Remove MEM_TYPE_ZERO_COPY in favor of the new MEM_TYPE_XSK_BUFF_POOL
APIs.
Cc: intel-wired-lan@lists.osuosl.org
Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
drivers/net/ethernet/intel/ice/ice_base.c | 16 +-
drivers/net/ethernet/intel/ice/ice_txrx.h | 8 +-
drivers/net/ethernet/intel/ice/ice_xsk.c | 372 +++-------------------
drivers/net/ethernet/intel/ice/ice_xsk.h | 13 +-
4 files changed, 54 insertions(+), 355 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c
index a19cd6f5436b..433eb72b1c85 100644
--- a/drivers/net/ethernet/intel/ice/ice_base.c
+++ b/drivers/net/ethernet/intel/ice/ice_base.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019, Intel Corporation. */
+#include <net/xdp_sock_drv.h>
#include "ice_base.h"
#include "ice_dcb_lib.h"
@@ -308,24 +309,23 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
if (ring->xsk_umem) {
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
- ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
- XDP_PACKET_HEADROOM;
+ ring->rx_buf_len =
+ xsk_umem_get_rx_frame_size(ring->xsk_umem);
/* For AF_XDP ZC, we disallow packets to span on
* multiple buffers, thus letting us skip that
* handling in the fast-path.
*/
chain_len = 1;
- ring->zca.free = ice_zca_free;
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
- MEM_TYPE_ZERO_COPY,
- &ring->zca);
+ MEM_TYPE_XSK_BUFF_POOL,
+ NULL);
if (err)
return err;
+ xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq);
- dev_info(ice_pf_to_dev(vsi->back), "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
+ dev_info(ice_pf_to_dev(vsi->back), "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
ring->q_index);
} else {
- ring->zca.free = NULL;
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq))
/* coverity[check_return] */
xdp_rxq_info_reg(&ring->xdp_rxq,
@@ -426,7 +426,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
writel(0, ring->tail);
err = ring->xsk_umem ?
- ice_alloc_rx_bufs_slow_zc(ring, ICE_DESC_UNUSED(ring)) :
+ ice_alloc_rx_bufs_zc(ring, ICE_DESC_UNUSED(ring)) :
ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));
if (err)
dev_info(ice_pf_to_dev(vsi->back), "Failed allocate some buffers on %sRx ring %d (pf_q %d)\n",
diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h
index 7ee00a128663..d0fd2173854f 100644
--- a/drivers/net/ethernet/intel/ice/ice_txrx.h
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.h
@@ -155,17 +155,16 @@ struct ice_tx_offload_params {
};
struct ice_rx_buf {
- struct sk_buff *skb;
- dma_addr_t dma;
union {
struct {
+ struct sk_buff *skb;
+ dma_addr_t dma;
struct page *page;
unsigned int page_offset;
u16 pagecnt_bias;
};
struct {
- void *addr;
- u64 handle;
+ struct xdp_buff *xdp;
};
};
};
@@ -289,7 +288,6 @@ struct ice_ring {
struct rcu_head rcu; /* to avoid race on free */
struct bpf_prog *xdp_prog;
struct xdp_umem *xsk_umem;
- struct zero_copy_allocator zca;
/* CL3 - 3rd cacheline starts here */
struct xdp_rxq_info xdp_rxq;
/* CLX - the below items are only accessed infrequently and should be
diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c
index 955b0fbb7c9a..da89589c3137 100644
--- a/drivers/net/ethernet/intel/ice/ice_xsk.c
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.c
@@ -279,28 +279,6 @@ static int ice_xsk_alloc_umems(struct ice_vsi *vsi)
return 0;
}
-/**
- * ice_xsk_add_umem - add a UMEM region for XDP sockets
- * @vsi: VSI to which the UMEM will be added
- * @umem: pointer to a requested UMEM region
- * @qid: queue ID
- *
- * Returns 0 on success, negative on error
- */
-static int ice_xsk_add_umem(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
-{
- int err;
-
- err = ice_xsk_alloc_umems(vsi);
- if (err)
- return err;
-
- vsi->xsk_umems[qid] = umem;
- vsi->num_xsk_umems_used++;
-
- return 0;
-}
-
/**
* ice_xsk_remove_umem - Remove an UMEM for a certain ring/qid
* @vsi: VSI from which the VSI will be removed
@@ -318,65 +296,6 @@ static void ice_xsk_remove_umem(struct ice_vsi *vsi, u16 qid)
}
}
-/**
- * ice_xsk_umem_dma_map - DMA map UMEM region for XDP sockets
- * @vsi: VSI to map the UMEM region
- * @umem: UMEM to map
- *
- * Returns 0 on success, negative on error
- */
-static int ice_xsk_umem_dma_map(struct ice_vsi *vsi, struct xdp_umem *umem)
-{
- struct ice_pf *pf = vsi->back;
- struct device *dev;
- unsigned int i;
-
- dev = ice_pf_to_dev(pf);
- for (i = 0; i < umem->npgs; i++) {
- dma_addr_t dma = dma_map_page_attrs(dev, umem->pgs[i], 0,
- PAGE_SIZE,
- DMA_BIDIRECTIONAL,
- ICE_RX_DMA_ATTR);
- if (dma_mapping_error(dev, dma)) {
- dev_dbg(dev, "XSK UMEM DMA mapping error on page num %d\n",
- i);
- goto out_unmap;
- }
-
- umem->pages[i].dma = dma;
- }
-
- return 0;
-
-out_unmap:
- for (; i > 0; i--) {
- dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);
- umem->pages[i].dma = 0;
- }
-
- return -EFAULT;
-}
-
-/**
- * ice_xsk_umem_dma_unmap - DMA unmap UMEM region for XDP sockets
- * @vsi: VSI from which the UMEM will be unmapped
- * @umem: UMEM to unmap
- */
-static void ice_xsk_umem_dma_unmap(struct ice_vsi *vsi, struct xdp_umem *umem)
-{
- struct ice_pf *pf = vsi->back;
- struct device *dev;
- unsigned int i;
-
- dev = ice_pf_to_dev(pf);
- for (i = 0; i < umem->npgs; i++) {
- dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);
-
- umem->pages[i].dma = 0;
- }
-}
/**
* ice_xsk_umem_disable - disable a UMEM region
@@ -391,7 +310,7 @@ static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid)
!vsi->xsk_umems[qid])
return -EINVAL;
- ice_xsk_umem_dma_unmap(vsi, vsi->xsk_umems[qid]);
+ xsk_buff_dma_unmap(vsi->xsk_umems[qid], ICE_RX_DMA_ATTR);
ice_xsk_remove_umem(vsi, qid);
return 0;
@@ -408,7 +327,6 @@ static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid)
static int
ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
{
- struct xdp_umem_fq_reuse *reuseq;
int err;
if (vsi->type != ICE_VSI_PF)
@@ -419,20 +337,18 @@ ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
if (qid >= vsi->num_xsk_umems)
return -EINVAL;
+ err = ice_xsk_alloc_umems(vsi);
+ if (err)
+ return err;
+
if (vsi->xsk_umems && vsi->xsk_umems[qid])
return -EBUSY;
- reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
- if (!reuseq)
- return -ENOMEM;
-
- xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
-
- err = ice_xsk_umem_dma_map(vsi, umem);
- if (err)
- return err;
+ vsi->xsk_umems[qid] = umem;
+ vsi->num_xsk_umems_used++;
- err = ice_xsk_add_umem(vsi, umem, qid);
+ err = xsk_buff_dma_map(vsi->xsk_umems[qid], ice_pf_to_dev(vsi->back),
+ ICE_RX_DMA_ATTR);
if (err)
return err;
@@ -483,119 +399,6 @@ int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
return ret;
}
-/**
- * ice_zca_free - Callback for MEM_TYPE_ZERO_COPY allocations
- * @zca: zero-cpoy allocator
- * @handle: Buffer handle
- */
-void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
-{
- struct ice_rx_buf *rx_buf;
- struct ice_ring *rx_ring;
- struct xdp_umem *umem;
- u64 hr, mask;
- u16 nta;
-
- rx_ring = container_of(zca, struct ice_ring, zca);
- umem = rx_ring->xsk_umem;
- hr = umem->headroom + XDP_PACKET_HEADROOM;
-
- mask = umem->chunk_mask;
-
- nta = rx_ring->next_to_alloc;
- rx_buf = &rx_ring->rx_buf[nta];
-
- nta++;
- rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
-
- handle &= mask;
-
- rx_buf->dma = xdp_umem_get_dma(umem, handle);
- rx_buf->dma += hr;
-
- rx_buf->addr = xdp_umem_get_data(umem, handle);
- rx_buf->addr += hr;
-
- rx_buf->handle = (u64)handle + umem->headroom;
-}
-
-/**
- * ice_alloc_buf_fast_zc - Retrieve buffer address from XDP umem
- * @rx_ring: ring with an xdp_umem bound to it
- * @rx_buf: buffer to which xsk page address will be assigned
- *
- * This function allocates an Rx buffer in the hot path.
- * The buffer can come from fill queue or recycle queue.
- *
- * Returns true if an assignment was successful, false if not.
- */
-static __always_inline bool
-ice_alloc_buf_fast_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
-{
- struct xdp_umem *umem = rx_ring->xsk_umem;
- void *addr = rx_buf->addr;
- u64 handle, hr;
-
- if (addr) {
- rx_ring->rx_stats.page_reuse_count++;
- return true;
- }
-
- if (!xsk_umem_peek_addr(umem, &handle)) {
- rx_ring->rx_stats.alloc_page_failed++;
- return false;
- }
-
- hr = umem->headroom + XDP_PACKET_HEADROOM;
-
- rx_buf->dma = xdp_umem_get_dma(umem, handle);
- rx_buf->dma += hr;
-
- rx_buf->addr = xdp_umem_get_data(umem, handle);
- rx_buf->addr += hr;
-
- rx_buf->handle = handle + umem->headroom;
-
- xsk_umem_release_addr(umem);
- return true;
-}
-
-/**
- * ice_alloc_buf_slow_zc - Retrieve buffer address from XDP umem
- * @rx_ring: ring with an xdp_umem bound to it
- * @rx_buf: buffer to which xsk page address will be assigned
- *
- * This function allocates an Rx buffer in the slow path.
- * The buffer can come from fill queue or recycle queue.
- *
- * Returns true if an assignment was successful, false if not.
- */
-static __always_inline bool
-ice_alloc_buf_slow_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
-{
- struct xdp_umem *umem = rx_ring->xsk_umem;
- u64 handle, headroom;
-
- if (!xsk_umem_peek_addr_rq(umem, &handle)) {
- rx_ring->rx_stats.alloc_page_failed++;
- return false;
- }
-
- handle &= umem->chunk_mask;
- headroom = umem->headroom + XDP_PACKET_HEADROOM;
-
- rx_buf->dma = xdp_umem_get_dma(umem, handle);
- rx_buf->dma += headroom;
-
- rx_buf->addr = xdp_umem_get_data(umem, handle);
- rx_buf->addr += headroom;
-
- rx_buf->handle = handle + umem->headroom;
-
- xsk_umem_release_addr_rq(umem);
- return true;
-}
-
/**
* ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
* @rx_ring: Rx ring
@@ -607,14 +410,13 @@ ice_alloc_buf_slow_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
*
* Returns false if all allocations were successful, true if any fail.
*/
-static bool
-ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
- bool (*alloc)(struct ice_ring *, struct ice_rx_buf *))
+bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count)
{
union ice_32b_rx_flex_desc *rx_desc;
u16 ntu = rx_ring->next_to_use;
struct ice_rx_buf *rx_buf;
bool ret = false;
+ dma_addr_t dma;
if (!count)
return false;
@@ -623,16 +425,14 @@ ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
rx_buf = &rx_ring->rx_buf[ntu];
do {
- if (!alloc(rx_ring, rx_buf)) {
+ rx_buf->xdp = xsk_buff_alloc(rx_ring->xsk_umem);
+ if (!rx_buf->xdp) {
ret = true;
break;
}
- dma_sync_single_range_for_device(rx_ring->dev, rx_buf->dma, 0,
- rx_ring->rx_buf_len,
- DMA_BIDIRECTIONAL);
-
- rx_desc->read.pkt_addr = cpu_to_le64(rx_buf->dma);
+ dma = xsk_buff_xdp_get_dma(rx_buf->xdp);
+ rx_desc->read.pkt_addr = cpu_to_le64(dma);
rx_desc->wb.status_error0 = 0;
rx_desc++;
@@ -652,32 +452,6 @@ ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
return ret;
}
-/**
- * ice_alloc_rx_bufs_fast_zc - allocate zero copy bufs in the hot path
- * @rx_ring: Rx ring
- * @count: number of bufs to allocate
- *
- * Returns false on success, true on failure.
- */
-static bool ice_alloc_rx_bufs_fast_zc(struct ice_ring *rx_ring, u16 count)
-{
- return ice_alloc_rx_bufs_zc(rx_ring, count,
- ice_alloc_buf_fast_zc);
-}
-
-/**
- * ice_alloc_rx_bufs_slow_zc - allocate zero copy bufs in the slow path
- * @rx_ring: Rx ring
- * @count: number of bufs to allocate
- *
- * Returns false on success, true on failure.
- */
-bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count)
-{
- return ice_alloc_rx_bufs_zc(rx_ring, count,
- ice_alloc_buf_slow_zc);
-}
-
/**
* ice_bump_ntc - Bump the next_to_clean counter of an Rx ring
* @rx_ring: Rx ring
@@ -691,59 +465,6 @@ static void ice_bump_ntc(struct ice_ring *rx_ring)
prefetch(ICE_RX_DESC(rx_ring, ntc));
}
-/**
- * ice_get_rx_buf_zc - Fetch the current Rx buffer
- * @rx_ring: Rx ring
- * @size: size of a buffer
- *
- * This function returns the current, received Rx buffer and does
- * DMA synchronization.
- *
- * Returns a pointer to the received Rx buffer.
- */
-static struct ice_rx_buf *ice_get_rx_buf_zc(struct ice_ring *rx_ring, int size)
-{
- struct ice_rx_buf *rx_buf;
-
- rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
-
- dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 0,
- size, DMA_BIDIRECTIONAL);
-
- return rx_buf;
-}
-
-/**
- * ice_reuse_rx_buf_zc - reuse an Rx buffer
- * @rx_ring: Rx ring
- * @old_buf: The buffer to recycle
- *
- * This function recycles a finished Rx buffer, and places it on the recycle
- * queue (next_to_alloc).
- */
-static void
-ice_reuse_rx_buf_zc(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf)
-{
- unsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask;
- u64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
- u16 nta = rx_ring->next_to_alloc;
- struct ice_rx_buf *new_buf;
-
- new_buf = &rx_ring->rx_buf[nta++];
- rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
-
- new_buf->dma = old_buf->dma & mask;
- new_buf->dma += hr;
-
- new_buf->addr = (void *)((unsigned long)old_buf->addr & mask);
- new_buf->addr += hr;
-
- new_buf->handle = old_buf->handle & mask;
- new_buf->handle += rx_ring->xsk_umem->headroom;
-
- old_buf->addr = NULL;
-}
-
/**
* ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
* @rx_ring: Rx ring
@@ -755,13 +476,12 @@ ice_reuse_rx_buf_zc(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf)
* Returns the skb on success, NULL on failure.
*/
static struct sk_buff *
-ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
- struct xdp_buff *xdp)
+ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
{
- unsigned int metasize = xdp->data - xdp->data_meta;
- unsigned int datasize = xdp->data_end - xdp->data;
- unsigned int datasize_hard = xdp->data_end -
- xdp->data_hard_start;
+ unsigned int metasize = rx_buf->xdp->data - rx_buf->xdp->data_meta;
+ unsigned int datasize = rx_buf->xdp->data_end - rx_buf->xdp->data;
+ unsigned int datasize_hard = rx_buf->xdp->data_end -
+ rx_buf->xdp->data_hard_start;
struct sk_buff *skb;
skb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard,
@@ -769,13 +489,13 @@ ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
if (unlikely(!skb))
return NULL;
- skb_reserve(skb, xdp->data - xdp->data_hard_start);
- memcpy(__skb_put(skb, datasize), xdp->data, datasize);
+ skb_reserve(skb, rx_buf->xdp->data - rx_buf->xdp->data_hard_start);
+ memcpy(__skb_put(skb, datasize), rx_buf->xdp->data, datasize);
if (metasize)
skb_metadata_set(skb, metasize);
- ice_reuse_rx_buf_zc(rx_ring, rx_buf);
-
+ xsk_buff_free(rx_buf->xdp);
+ rx_buf->xdp = NULL;
return skb;
}
@@ -802,7 +522,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)
}
act = bpf_prog_run_xdp(xdp_prog, xdp);
- xdp->handle += xdp->data - xdp->data_hard_start;
switch (act) {
case XDP_PASS:
break;
@@ -842,9 +561,6 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
u16 cleaned_count = ICE_DESC_UNUSED(rx_ring);
unsigned int xdp_xmit = 0;
bool failure = false;
- struct xdp_buff xdp;
-
- xdp.rxq = &rx_ring->xdp_rxq;
while (likely(total_rx_packets < (unsigned int)budget)) {
union ice_32b_rx_flex_desc *rx_desc;
@@ -856,8 +572,8 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
u8 rx_ptype;
if (cleaned_count >= ICE_RX_BUF_WRITE) {
- failure |= ice_alloc_rx_bufs_fast_zc(rx_ring,
- cleaned_count);
+ failure |= ice_alloc_rx_bufs_zc(rx_ring,
+ cleaned_count);
cleaned_count = 0;
}
@@ -878,25 +594,19 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
if (!size)
break;
- rx_buf = ice_get_rx_buf_zc(rx_ring, size);
- if (!rx_buf->addr)
- break;
- xdp.data = rx_buf->addr;
- xdp.data_meta = xdp.data;
- xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
- xdp.data_end = xdp.data + size;
- xdp.handle = rx_buf->handle;
+ rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
+ rx_buf->xdp->data_end = rx_buf->xdp->data + size;
+ xsk_buff_dma_sync_for_cpu(rx_buf->xdp);
- xdp_res = ice_run_xdp_zc(rx_ring, &xdp);
+ xdp_res = ice_run_xdp_zc(rx_ring, rx_buf->xdp);
if (xdp_res) {
- if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) {
+ if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))
xdp_xmit |= xdp_res;
- rx_buf->addr = NULL;
- } else {
- ice_reuse_rx_buf_zc(rx_ring, rx_buf);
- }
+ else
+ xsk_buff_free(rx_buf->xdp);
+ rx_buf->xdp = NULL;
total_rx_bytes += size;
total_rx_packets++;
cleaned_count++;
@@ -906,7 +616,7 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
}
/* XDP_PASS path */
- skb = ice_construct_skb_zc(rx_ring, rx_buf, &xdp);
+ skb = ice_construct_skb_zc(rx_ring, rx_buf);
if (!skb) {
rx_ring->rx_stats.alloc_buf_failed++;
break;
@@ -977,10 +687,9 @@ static bool ice_xmit_zc(struct ice_ring *xdp_ring, int budget)
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
break;
- dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
-
- dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
- DMA_BIDIRECTIONAL);
+ dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr);
+ xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma,
+ desc.len);
tx_buf->bytecount = desc.len;
@@ -1163,11 +872,10 @@ void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring)
for (i = 0; i < rx_ring->count; i++) {
struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];
- if (!rx_buf->addr)
+ if (!rx_buf->xdp)
continue;
- xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_buf->handle);
- rx_buf->addr = NULL;
+ rx_buf->xdp = NULL;
}
}
diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h
index 8a4ba7c6d549..fc1a06b4df36 100644
--- a/drivers/net/ethernet/intel/ice/ice_xsk.h
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.h
@@ -10,11 +10,10 @@ struct ice_vsi;
#ifdef CONFIG_XDP_SOCKETS
int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid);
-void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle);
int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget);
bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget);
int ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags);
-bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count);
+bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count);
bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi);
void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring);
void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring);
@@ -27,12 +26,6 @@ ice_xsk_umem_setup(struct ice_vsi __always_unused *vsi,
return -EOPNOTSUPP;
}
-static inline void
-ice_zca_free(struct zero_copy_allocator __always_unused *zca,
- unsigned long __always_unused handle)
-{
-}
-
static inline int
ice_clean_rx_irq_zc(struct ice_ring __always_unused *rx_ring,
int __always_unused budget)
@@ -48,8 +41,8 @@ ice_clean_tx_irq_zc(struct ice_ring __always_unused *xdp_ring,
}
static inline bool
-ice_alloc_rx_bufs_slow_zc(struct ice_ring __always_unused *rx_ring,
- u16 __always_unused count)
+ice_alloc_rx_bufs_zc(struct ice_ring __always_unused *rx_ring,
+ u16 __always_unused count)
{
return false;
}
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 09/14] ixgbe, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (7 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 08/14] ice, " Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 10/14] mlx5, " Björn Töpel
` (4 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski, intel-wired-lan
From: Björn Töpel <bjorn.topel@intel.com>
Remove MEM_TYPE_ZERO_COPY in favor of the new MEM_TYPE_XSK_BUFF_POOL
APIs.
Cc: intel-wired-lan@lists.osuosl.org
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
drivers/net/ethernet/intel/ixgbe/ixgbe.h | 9 +-
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 15 +-
.../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 2 +-
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 305 +++---------------
4 files changed, 62 insertions(+), 269 deletions(-)
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index 2833e4f041ce..5ddfc83a1e46 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -224,17 +224,17 @@ struct ixgbe_tx_buffer {
};
struct ixgbe_rx_buffer {
- struct sk_buff *skb;
- dma_addr_t dma;
union {
struct {
+ struct sk_buff *skb;
+ dma_addr_t dma;
struct page *page;
__u32 page_offset;
__u16 pagecnt_bias;
};
struct {
- void *addr;
- u64 handle;
+ bool discard;
+ struct xdp_buff *xdp;
};
};
};
@@ -351,7 +351,6 @@ struct ixgbe_ring {
};
struct xdp_rxq_info xdp_rxq;
struct xdp_umem *xsk_umem;
- struct zero_copy_allocator zca; /* ZC allocator anchor */
u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */
u16 rx_buf_len;
} ____cacheline_internodealigned_in_smp;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 718931d951bc..da7b8042901f 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -35,7 +35,7 @@
#include <net/tc_act/tc_mirred.h>
#include <net/vxlan.h>
#include <net/mpls.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include <net/xfrm.h>
#include "ixgbe.h"
@@ -3726,8 +3726,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
/* configure the packet buffer length */
if (rx_ring->xsk_umem) {
- u32 xsk_buf_len = rx_ring->xsk_umem->chunk_size_nohr -
- XDP_PACKET_HEADROOM;
+ u32 xsk_buf_len = xsk_umem_get_rx_frame_size(rx_ring->xsk_umem);
/* If the MAC support setting RXDCTL.RLPML, the
* SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and
@@ -4074,11 +4073,10 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
ring->xsk_umem = ixgbe_xsk_umem(adapter, ring);
if (ring->xsk_umem) {
- ring->zca.free = ixgbe_zca_free;
WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
- MEM_TYPE_ZERO_COPY,
- &ring->zca));
-
+ MEM_TYPE_XSK_BUFF_POOL,
+ NULL));
+ xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq);
} else {
WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
MEM_TYPE_PAGE_SHARED, NULL));
@@ -4134,8 +4132,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
}
if (ring->xsk_umem && hw->mac.type != ixgbe_mac_82599EB) {
- u32 xsk_buf_len = ring->xsk_umem->chunk_size_nohr -
- XDP_PACKET_HEADROOM;
+ u32 xsk_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem);
rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
IXGBE_RXDCTL_RLPML_EN);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h
index 6d01700b46bc..7887ae4aaf4f 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h
@@ -35,7 +35,7 @@ int ixgbe_xsk_umem_setup(struct ixgbe_adapter *adapter, struct xdp_umem *umem,
void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);
-void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count);
+bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count);
int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
struct ixgbe_ring *rx_ring,
const int budget);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
index 5b6edbd8a4ed..12f5dc3fe76e 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
@@ -20,54 +20,11 @@ struct xdp_umem *ixgbe_xsk_umem(struct ixgbe_adapter *adapter,
return xdp_get_umem_from_qid(adapter->netdev, qid);
}
-static int ixgbe_xsk_umem_dma_map(struct ixgbe_adapter *adapter,
- struct xdp_umem *umem)
-{
- struct device *dev = &adapter->pdev->dev;
- unsigned int i, j;
- dma_addr_t dma;
-
- for (i = 0; i < umem->npgs; i++) {
- dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
- if (dma_mapping_error(dev, dma))
- goto out_unmap;
-
- umem->pages[i].dma = dma;
- }
-
- return 0;
-
-out_unmap:
- for (j = 0; j < i; j++) {
- dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
- umem->pages[i].dma = 0;
- }
-
- return -1;
-}
-
-static void ixgbe_xsk_umem_dma_unmap(struct ixgbe_adapter *adapter,
- struct xdp_umem *umem)
-{
- struct device *dev = &adapter->pdev->dev;
- unsigned int i;
-
- for (i = 0; i < umem->npgs; i++) {
- dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
-
- umem->pages[i].dma = 0;
- }
-}
-
static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter,
struct xdp_umem *umem,
u16 qid)
{
struct net_device *netdev = adapter->netdev;
- struct xdp_umem_fq_reuse *reuseq;
bool if_running;
int err;
@@ -78,13 +35,7 @@ static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter,
qid >= netdev->real_num_tx_queues)
return -EINVAL;
- reuseq = xsk_reuseq_prepare(adapter->rx_ring[0]->count);
- if (!reuseq)
- return -ENOMEM;
-
- xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
-
- err = ixgbe_xsk_umem_dma_map(adapter, umem);
+ err = xsk_buff_dma_map(umem, &adapter->pdev->dev, IXGBE_RX_DMA_ATTR);
if (err)
return err;
@@ -124,7 +75,7 @@ static int ixgbe_xsk_umem_disable(struct ixgbe_adapter *adapter, u16 qid)
ixgbe_txrx_ring_disable(adapter, qid);
clear_bit(qid, adapter->af_xdp_zc_qps);
- ixgbe_xsk_umem_dma_unmap(adapter, umem);
+ xsk_buff_dma_unmap(umem, IXGBE_RX_DMA_ATTR);
if (if_running)
ixgbe_txrx_ring_enable(adapter, qid);
@@ -143,19 +94,14 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter,
struct ixgbe_ring *rx_ring,
struct xdp_buff *xdp)
{
- struct xdp_umem *umem = rx_ring->xsk_umem;
int err, result = IXGBE_XDP_PASS;
struct bpf_prog *xdp_prog;
struct xdp_frame *xdpf;
- u64 offset;
u32 act;
rcu_read_lock();
xdp_prog = READ_ONCE(rx_ring->xdp_prog);
act = bpf_prog_run_xdp(xdp_prog, xdp);
- offset = xdp->data - xdp->data_hard_start;
-
- xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset);
switch (act) {
case XDP_PASS:
@@ -186,140 +132,16 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter,
return result;
}
-static struct
-ixgbe_rx_buffer *ixgbe_get_rx_buffer_zc(struct ixgbe_ring *rx_ring,
- unsigned int size)
-{
- struct ixgbe_rx_buffer *bi;
-
- bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
-
- /* we are reusing so sync this buffer for CPU use */
- dma_sync_single_range_for_cpu(rx_ring->dev,
- bi->dma, 0,
- size,
- DMA_BIDIRECTIONAL);
-
- return bi;
-}
-
-static void ixgbe_reuse_rx_buffer_zc(struct ixgbe_ring *rx_ring,
- struct ixgbe_rx_buffer *obi)
-{
- u16 nta = rx_ring->next_to_alloc;
- struct ixgbe_rx_buffer *nbi;
-
- nbi = &rx_ring->rx_buffer_info[rx_ring->next_to_alloc];
- /* update, and store next to alloc */
- nta++;
- rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
-
- /* transfer page from old buffer to new buffer */
- nbi->dma = obi->dma;
- nbi->addr = obi->addr;
- nbi->handle = obi->handle;
-
- obi->addr = NULL;
- obi->skb = NULL;
-}
-
-void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
-{
- struct ixgbe_rx_buffer *bi;
- struct ixgbe_ring *rx_ring;
- u64 hr, mask;
- u16 nta;
-
- rx_ring = container_of(alloc, struct ixgbe_ring, zca);
- hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
- mask = rx_ring->xsk_umem->chunk_mask;
-
- nta = rx_ring->next_to_alloc;
- bi = rx_ring->rx_buffer_info;
-
- nta++;
- rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
-
- handle &= mask;
-
- bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle);
- bi->dma += hr;
-
- bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle);
- bi->addr += hr;
-
- bi->handle = xsk_umem_adjust_offset(rx_ring->xsk_umem, (u64)handle,
- rx_ring->xsk_umem->headroom);
-}
-
-static bool ixgbe_alloc_buffer_zc(struct ixgbe_ring *rx_ring,
- struct ixgbe_rx_buffer *bi)
-{
- struct xdp_umem *umem = rx_ring->xsk_umem;
- void *addr = bi->addr;
- u64 handle, hr;
-
- if (addr)
- return true;
-
- if (!xsk_umem_peek_addr(umem, &handle)) {
- rx_ring->rx_stats.alloc_rx_page_failed++;
- return false;
- }
-
- hr = umem->headroom + XDP_PACKET_HEADROOM;
-
- bi->dma = xdp_umem_get_dma(umem, handle);
- bi->dma += hr;
-
- bi->addr = xdp_umem_get_data(umem, handle);
- bi->addr += hr;
-
- bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
-
- xsk_umem_release_addr(umem);
- return true;
-}
-
-static bool ixgbe_alloc_buffer_slow_zc(struct ixgbe_ring *rx_ring,
- struct ixgbe_rx_buffer *bi)
-{
- struct xdp_umem *umem = rx_ring->xsk_umem;
- u64 handle, hr;
-
- if (!xsk_umem_peek_addr_rq(umem, &handle)) {
- rx_ring->rx_stats.alloc_rx_page_failed++;
- return false;
- }
-
- handle &= rx_ring->xsk_umem->chunk_mask;
-
- hr = umem->headroom + XDP_PACKET_HEADROOM;
-
- bi->dma = xdp_umem_get_dma(umem, handle);
- bi->dma += hr;
-
- bi->addr = xdp_umem_get_data(umem, handle);
- bi->addr += hr;
-
- bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
-
- xsk_umem_release_addr_rq(umem);
- return true;
-}
-
-static __always_inline bool
-__ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
- bool alloc(struct ixgbe_ring *rx_ring,
- struct ixgbe_rx_buffer *bi))
+bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count)
{
union ixgbe_adv_rx_desc *rx_desc;
struct ixgbe_rx_buffer *bi;
u16 i = rx_ring->next_to_use;
+ dma_addr_t dma;
bool ok = true;
/* nothing to do */
- if (!cleaned_count)
+ if (!count)
return true;
rx_desc = IXGBE_RX_DESC(rx_ring, i);
@@ -327,21 +149,18 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
i -= rx_ring->count;
do {
- if (!alloc(rx_ring, bi)) {
+ bi->xdp = xsk_buff_alloc(rx_ring->xsk_umem);
+ if (!bi->xdp) {
ok = false;
break;
}
- /* sync the buffer for use by the device */
- dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
- bi->page_offset,
- rx_ring->rx_buf_len,
- DMA_BIDIRECTIONAL);
+ dma = xsk_buff_xdp_get_dma(bi->xdp);
/* Refresh the desc even if buffer_addrs didn't change
* because each write-back erases this info.
*/
- rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
+ rx_desc->read.pkt_addr = cpu_to_le64(dma);
rx_desc++;
bi++;
@@ -355,17 +174,14 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
/* clear the length for the next_to_use descriptor */
rx_desc->wb.upper.length = 0;
- cleaned_count--;
- } while (cleaned_count);
+ count--;
+ } while (count);
i += rx_ring->count;
if (rx_ring->next_to_use != i) {
rx_ring->next_to_use = i;
- /* update next to alloc since we have filled the ring */
- rx_ring->next_to_alloc = i;
-
/* Force memory writes to complete before letting h/w
* know there are new descriptors to fetch. (Only
* applicable for weak-ordered memory model archs,
@@ -378,40 +194,27 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
return ok;
}
-void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count)
-{
- __ixgbe_alloc_rx_buffers_zc(rx_ring, count,
- ixgbe_alloc_buffer_slow_zc);
-}
-
-static bool ixgbe_alloc_rx_buffers_fast_zc(struct ixgbe_ring *rx_ring,
- u16 count)
-{
- return __ixgbe_alloc_rx_buffers_zc(rx_ring, count,
- ixgbe_alloc_buffer_zc);
-}
-
static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring,
- struct ixgbe_rx_buffer *bi,
- struct xdp_buff *xdp)
+ struct ixgbe_rx_buffer *bi)
{
- unsigned int metasize = xdp->data - xdp->data_meta;
- unsigned int datasize = xdp->data_end - xdp->data;
+ unsigned int metasize = bi->xdp->data - bi->xdp->data_meta;
+ unsigned int datasize = bi->xdp->data_end - bi->xdp->data;
struct sk_buff *skb;
/* allocate a skb to store the frags */
skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
- xdp->data_end - xdp->data_hard_start,
+ bi->xdp->data_end - bi->xdp->data_hard_start,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb))
return NULL;
- skb_reserve(skb, xdp->data - xdp->data_hard_start);
- memcpy(__skb_put(skb, datasize), xdp->data, datasize);
+ skb_reserve(skb, bi->xdp->data - bi->xdp->data_hard_start);
+ memcpy(__skb_put(skb, datasize), bi->xdp->data, datasize);
if (metasize)
skb_metadata_set(skb, metasize);
- ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
+ xsk_buff_free(bi->xdp);
+ bi->xdp = NULL;
return skb;
}
@@ -434,9 +237,6 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
unsigned int xdp_res, xdp_xmit = 0;
bool failure = false;
struct sk_buff *skb;
- struct xdp_buff xdp;
-
- xdp.rxq = &rx_ring->xdp_rxq;
while (likely(total_rx_packets < budget)) {
union ixgbe_adv_rx_desc *rx_desc;
@@ -446,8 +246,8 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
/* return some buffers to hardware, one at a time is too slow */
if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
failure = failure ||
- !ixgbe_alloc_rx_buffers_fast_zc(rx_ring,
- cleaned_count);
+ !ixgbe_alloc_rx_buffers_zc(rx_ring,
+ cleaned_count);
cleaned_count = 0;
}
@@ -462,42 +262,40 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
*/
dma_rmb();
- bi = ixgbe_get_rx_buffer_zc(rx_ring, size);
+ bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
if (unlikely(!ixgbe_test_staterr(rx_desc,
IXGBE_RXD_STAT_EOP))) {
struct ixgbe_rx_buffer *next_bi;
- ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
+ xsk_buff_free(bi->xdp);
+ bi->xdp = NULL;
ixgbe_inc_ntc(rx_ring);
next_bi =
&rx_ring->rx_buffer_info[rx_ring->next_to_clean];
- next_bi->skb = ERR_PTR(-EINVAL);
+ next_bi->discard = true;
continue;
}
- if (unlikely(bi->skb)) {
- ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
+ if (unlikely(bi->discard)) {
+ xsk_buff_free(bi->xdp);
+ bi->xdp = NULL;
+ bi->discard = false;
ixgbe_inc_ntc(rx_ring);
continue;
}
- xdp.data = bi->addr;
- xdp.data_meta = xdp.data;
- xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
- xdp.data_end = xdp.data + size;
- xdp.handle = bi->handle;
-
- xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &xdp);
+ bi->xdp->data_end = bi->xdp->data_end + size;
+ xsk_buff_dma_sync_for_cpu(bi->xdp);
+ xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp);
if (xdp_res) {
- if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) {
+ if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))
xdp_xmit |= xdp_res;
- bi->addr = NULL;
- bi->skb = NULL;
- } else {
- ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
- }
+ else
+ xsk_buff_free(bi->xdp);
+
+ bi->xdp = NULL;
total_rx_packets++;
total_rx_bytes += size;
@@ -507,7 +305,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
}
/* XDP_PASS path */
- skb = ixgbe_construct_skb_zc(rx_ring, bi, &xdp);
+ skb = ixgbe_construct_skb_zc(rx_ring, bi);
if (!skb) {
rx_ring->rx_stats.alloc_rx_buff_failed++;
break;
@@ -559,17 +357,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring)
{
- u16 i = rx_ring->next_to_clean;
- struct ixgbe_rx_buffer *bi = &rx_ring->rx_buffer_info[i];
+ struct ixgbe_rx_buffer *bi;
+ u16 i;
- while (i != rx_ring->next_to_alloc) {
- xsk_umem_fq_reuse(rx_ring->xsk_umem, bi->handle);
- i++;
- bi++;
- if (i == rx_ring->count) {
- i = 0;
- bi = rx_ring->rx_buffer_info;
- }
+ for (i = 0; i < rx_ring->count; i++) {
+ bi = &rx_ring->rx_buffer_info[i];
+
+ if (!bi->xdp)
+ continue;
+
+ xsk_buff_free(bi->xdp);
+ bi->xdp = NULL;
}
}
@@ -592,10 +390,9 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
break;
- dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
-
- dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
- DMA_BIDIRECTIONAL);
+ dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr);
+ xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma,
+ desc.len);
tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];
tx_bi->bytecount = desc.len;
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (8 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 09/14] ixgbe, " Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-08 11:55 ` Maxim Mikityanskiy
2020-05-07 10:42 ` [PATCH bpf-next 11/14] xsk: remove MEM_TYPE_ZERO_COPY and corresponding code Björn Töpel
` (3 subsequent siblings)
13 siblings, 1 reply; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
Use the new MEM_TYPE_XSK_BUFF_POOL API in lieu of MEM_TYPE_ZERO_COPY in
mlx5e. It allows to drop a lot of code from the driver (which is now
common in AF_XDP core and was related to XSK RX frame allocation, DMA
mapping, etc.) and slightly improve performance.
rfc->v1: Put back the sanity check for XSK params, use XSK API to get
the total headroom size. (Maxim)
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
---
drivers/net/ethernet/mellanox/mlx5/core/en.h | 11 +-
.../ethernet/mellanox/mlx5/core/en/params.c | 13 +-
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 30 ++---
.../net/ethernet/mellanox/mlx5/core/en/xdp.h | 2 +-
.../ethernet/mellanox/mlx5/core/en/xsk/rx.c | 111 +++---------------
.../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 6 -
.../ethernet/mellanox/mlx5/core/en/xsk/tx.c | 6 +-
.../ethernet/mellanox/mlx5/core/en/xsk/umem.c | 49 +-------
.../net/ethernet/mellanox/mlx5/core/en_main.c | 15 +--
.../net/ethernet/mellanox/mlx5/core/en_rx.c | 43 +++++--
10 files changed, 84 insertions(+), 202 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 0864b76ca2c0..368563ff8efb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -426,13 +426,12 @@ struct mlx5e_txqsq {
} ____cacheline_aligned_in_smp;
struct mlx5e_dma_info {
- dma_addr_t addr;
union {
- struct page *page;
struct {
- u64 handle;
- void *data;
- } xsk;
+ dma_addr_t addr;
+ struct page *page;
+ };
+ struct xdp_buff *xsk;
};
};
@@ -650,7 +649,6 @@ struct mlx5e_rq {
} mpwqe;
};
struct {
- u16 umem_headroom;
u16 headroom;
u8 map_dir; /* dma map direction */
} buff;
@@ -682,7 +680,6 @@ struct mlx5e_rq {
struct page_pool *page_pool;
/* AF_XDP zero-copy */
- struct zero_copy_allocator zca;
struct xdp_umem *umem;
struct work_struct recover_work;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index eb2e1f2138e4..38e4f19d69f8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -12,15 +12,16 @@ static inline bool mlx5e_rx_is_xdp(struct mlx5e_params *params,
u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
{
- u16 headroom = NET_IP_ALIGN;
+ u16 headroom;
- if (mlx5e_rx_is_xdp(params, xsk)) {
+ if (xsk)
+ return xsk->headroom;
+
+ headroom = NET_IP_ALIGN;
+ if (mlx5e_rx_is_xdp(params, xsk))
headroom += XDP_PACKET_HEADROOM;
- if (xsk)
- headroom += xsk->headroom;
- } else {
+ else
headroom += MLX5_RX_HEADROOM;
- }
return headroom;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index b04b99396f65..a2a194525b15 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -71,7 +71,7 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
xdptxd.data = xdpf->data;
xdptxd.len = xdpf->len;
- if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) {
+ if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
/* The xdp_buff was in the UMEM and was copied into a newly
* allocated page. The UMEM page was returned via the ZCA, and
* this new page has to be mapped at this point and has to be
@@ -119,49 +119,33 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
/* returns true if packet was consumed by xdp */
bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
- void *va, u16 *rx_headroom, u32 *len, bool xsk)
+ u32 *len, struct xdp_buff *xdp)
{
struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
- struct xdp_umem *umem = rq->umem;
- struct xdp_buff xdp;
u32 act;
int err;
if (!prog)
return false;
- xdp.data = va + *rx_headroom;
- xdp_set_data_meta_invalid(&xdp);
- xdp.data_end = xdp.data + *len;
- xdp.data_hard_start = va;
- if (xsk)
- xdp.handle = di->xsk.handle;
- xdp.rxq = &rq->xdp_rxq;
-
- act = bpf_prog_run_xdp(prog, &xdp);
- if (xsk) {
- u64 off = xdp.data - xdp.data_hard_start;
-
- xdp.handle = xsk_umem_adjust_offset(umem, xdp.handle, off);
- }
+ act = bpf_prog_run_xdp(prog, xdp);
switch (act) {
case XDP_PASS:
- *rx_headroom = xdp.data - xdp.data_hard_start;
- *len = xdp.data_end - xdp.data;
+ *len = xdp->data_end - xdp->data;
return false;
case XDP_TX:
- if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, di, &xdp)))
+ if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, di, xdp)))
goto xdp_abort;
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
return true;
case XDP_REDIRECT:
/* When XDP enabled then page-refcnt==1 here */
- err = xdp_do_redirect(rq->netdev, &xdp, prog);
+ err = xdp_do_redirect(rq->netdev, xdp, prog);
if (unlikely(err))
goto xdp_abort;
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
__set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
- if (!xsk)
+ if (xdp->rxq->mem.type != MEM_TYPE_XSK_BUFF_POOL)
mlx5e_page_dma_unmap(rq, di);
rq->stats->xdp_redirect++;
return true;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
index ed6f045febeb..54bad625267f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
@@ -63,7 +63,7 @@
struct mlx5e_xsk_param;
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
- void *va, u16 *rx_headroom, u32 *len, bool xsk);
+ u32 *len, struct xdp_buff *xdp);
void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
index 62fc8a128a8d..59c40983d428 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
@@ -3,71 +3,10 @@
#include "rx.h"
#include "en/xdp.h"
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
/* RX data path */
-bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count)
-{
- /* Check in advance that we have enough frames, instead of allocating
- * one-by-one, failing and moving frames to the Reuse Ring.
- */
- return xsk_umem_has_addrs_rq(rq->umem, count);
-}
-
-int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info)
-{
- struct xdp_umem *umem = rq->umem;
- u64 handle;
-
- if (!xsk_umem_peek_addr_rq(umem, &handle))
- return -ENOMEM;
-
- dma_info->xsk.handle = xsk_umem_adjust_offset(umem, handle,
- rq->buff.umem_headroom);
- dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle);
-
- /* No need to add headroom to the DMA address. In striding RQ case, we
- * just provide pages for UMR, and headroom is counted at the setup
- * stage when creating a WQE. In non-striding RQ case, headroom is
- * accounted in mlx5e_alloc_rx_wqe.
- */
- dma_info->addr = xdp_umem_get_dma(umem, handle);
-
- xsk_umem_release_addr_rq(umem);
-
- dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
-
- return 0;
-}
-
-static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle)
-{
- xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask);
-}
-
-/* XSKRQ uses pages from UMEM, they must not be released. They are returned to
- * the userspace if possible, and if not, this function is called to reuse them
- * in the driver.
- */
-void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info)
-{
- mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle);
-}
-
-/* Return a frame back to the hardware to fill in again. It is used by XDP when
- * the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP.
- */
-void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
-{
- struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca);
-
- mlx5e_xsk_recycle_frame(rq, handle);
-}
-
static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data,
u32 cqe_bcnt)
{
@@ -90,11 +29,8 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
u32 head_offset,
u32 page_idx)
{
- struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
- u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
+ struct xdp_buff *xdp = wi->umr.dma_info[page_idx].xsk;
u32 cqe_bcnt32 = cqe_bcnt;
- void *va, *data;
- u32 frag_size;
bool consumed;
/* Check packet size. Note LRO doesn't use linear SKB */
@@ -103,22 +39,19 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
return NULL;
}
- /* head_offset is not used in this function, because di->xsk.data and
- * di->addr point directly to the necessary place. Furthermore, in the
- * current implementation, UMR pages are mapped to XSK frames, so
+ /* head_offset is not used in this function, because xdp->data and the
+ * DMA address point directly to the necessary place. Furthermore, in
+ * the current implementation, UMR pages are mapped to XSK frames, so
* head_offset should always be 0.
*/
WARN_ON_ONCE(head_offset);
- va = di->xsk.data;
- data = va + rx_headroom;
- frag_size = rq->buff.headroom + cqe_bcnt32;
-
- dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
- prefetch(data);
+ xdp->data_end = xdp->data + cqe_bcnt32;
+ xsk_buff_dma_sync_for_cpu(xdp);
+ prefetch(xdp->data);
rcu_read_lock();
- consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, true);
+ consumed = mlx5e_xdp_handle(rq, NULL, &cqe_bcnt32, xdp);
rcu_read_unlock();
/* Possible flows:
@@ -145,7 +78,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
/* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
* frame. On SKB allocation failure, NULL is returned.
*/
- return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt32);
+ return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt32);
}
struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
@@ -153,25 +86,19 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *wi,
u32 cqe_bcnt)
{
- struct mlx5e_dma_info *di = wi->di;
- u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
- void *va, *data;
+ struct xdp_buff *xdp = wi->di->xsk;
bool consumed;
- u32 frag_size;
- /* wi->offset is not used in this function, because di->xsk.data and
- * di->addr point directly to the necessary place. Furthermore, in the
- * current implementation, one page = one packet = one frame, so
+ /* wi->offset is not used in this function, because xdp->data and the
+ * DMA address point directly to the necessary place. Furthermore, the
+ * XSK allocator allocates frames per packet, instead of pages, so
* wi->offset should always be 0.
*/
WARN_ON_ONCE(wi->offset);
- va = di->xsk.data;
- data = va + rx_headroom;
- frag_size = rq->buff.headroom + cqe_bcnt;
-
- dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
- prefetch(data);
+ xdp->data_end = xdp->data + cqe_bcnt;
+ xsk_buff_dma_sync_for_cpu(xdp);
+ prefetch(xdp->data);
if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
rq->stats->wqe_err++;
@@ -179,7 +106,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
}
rcu_read_lock();
- consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, true);
+ consumed = mlx5e_xdp_handle(rq, NULL, &cqe_bcnt, xdp);
rcu_read_unlock();
if (likely(consumed))
@@ -189,5 +116,5 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
* will be handled by mlx5e_put_rx_frag.
* On SKB allocation failure, NULL is returned.
*/
- return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt);
+ return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
index a8e11adbf426..0062652f37b3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
@@ -9,12 +9,6 @@
/* RX data path */
-bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count);
-int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info);
-void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
- struct mlx5e_dma_info *dma_info);
-void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle);
struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
struct mlx5e_mpw_info *wi,
u16 cqe_bcnt,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
index 3bcdb5b2fc20..470f3b6317c3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
@@ -5,7 +5,7 @@
#include "umem.h"
#include "en/xdp.h"
#include "en/params.h"
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
{
@@ -92,8 +92,8 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
break;
}
- xdptxd.dma_addr = xdp_umem_get_dma(umem, desc.addr);
- xdptxd.data = xdp_umem_get_data(umem, desc.addr);
+ xdptxd.dma_addr = xsk_buff_raw_get_dma(umem, desc.addr);
+ xdptxd.data = xsk_buff_raw_get_data(umem, desc.addr);
xdptxd.len = desc.len;
dma_sync_single_for_device(sq->pdev, xdptxd.dma_addr,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
index 5e49fdb564b3..7b17fcd0a56d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
@@ -10,40 +10,14 @@ static int mlx5e_xsk_map_umem(struct mlx5e_priv *priv,
struct xdp_umem *umem)
{
struct device *dev = priv->mdev->device;
- u32 i;
- for (i = 0; i < umem->npgs; i++) {
- dma_addr_t dma = dma_map_page(dev, umem->pgs[i], 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
-
- if (unlikely(dma_mapping_error(dev, dma)))
- goto err_unmap;
- umem->pages[i].dma = dma;
- }
-
- return 0;
-
-err_unmap:
- while (i--) {
- dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- umem->pages[i].dma = 0;
- }
-
- return -ENOMEM;
+ return xsk_buff_dma_map(umem, dev, 0);
}
static void mlx5e_xsk_unmap_umem(struct mlx5e_priv *priv,
struct xdp_umem *umem)
{
- struct device *dev = priv->mdev->device;
- u32 i;
-
- for (i = 0; i < umem->npgs; i++) {
- dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- umem->pages[i].dma = 0;
- }
+ return xsk_buff_dma_unmap(umem, 0);
}
static int mlx5e_xsk_get_umems(struct mlx5e_xsk *xsk)
@@ -90,13 +64,14 @@ static void mlx5e_xsk_remove_umem(struct mlx5e_xsk *xsk, u16 ix)
static bool mlx5e_xsk_is_umem_sane(struct xdp_umem *umem)
{
- return umem->headroom <= 0xffff && umem->chunk_size_nohr <= 0xffff;
+ return xsk_umem_get_headroom(umem) <= 0xffff &&
+ xsk_umem_get_chunk_size(umem) <= 0xffff;
}
void mlx5e_build_xsk_param(struct xdp_umem *umem, struct mlx5e_xsk_param *xsk)
{
- xsk->headroom = umem->headroom;
- xsk->chunk_size = umem->chunk_size_nohr + umem->headroom;
+ xsk->headroom = xsk_umem_get_headroom(umem);
+ xsk->chunk_size = xsk_umem_get_chunk_size(umem);
}
static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
@@ -241,18 +216,6 @@ int mlx5e_xsk_setup_umem(struct net_device *dev, struct xdp_umem *umem, u16 qid)
mlx5e_xsk_disable_umem(priv, ix);
}
-int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries)
-{
- struct xdp_umem_fq_reuse *reuseq;
-
- reuseq = xsk_reuseq_prepare(nentries);
- if (unlikely(!reuseq))
- return -ENOMEM;
- xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
-
- return 0;
-}
-
u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk)
{
u16 res = xsk->refcnt ? params->num_channels : 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 048a4f8601a8..5ce73931eff4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -38,7 +38,7 @@
#include <linux/bpf.h>
#include <linux/if_bridge.h>
#include <net/page_pool.h>
-#include <net/xdp_sock.h>
+#include <net/xdp_sock_drv.h>
#include "eswitch.h"
#include "en.h"
#include "en/txrx.h"
@@ -414,7 +414,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
- rq->buff.umem_headroom = xsk ? xsk->headroom : 0;
pool_size = 1 << params->log_rq_mtu_frames;
switch (rq->wq_type) {
@@ -522,17 +521,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
}
if (xsk) {
- err = mlx5e_xsk_resize_reuseq(umem, num_xsk_frames);
- if (unlikely(err)) {
- mlx5_core_err(mdev, "Unable to allocate the Reuse Ring for %u frames\n",
- num_xsk_frames);
- goto err_free;
- }
-
- rq->zca.free = mlx5e_xsk_zca_free;
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
- MEM_TYPE_ZERO_COPY,
- &rq->zca);
+ MEM_TYPE_XSK_BUFF_POOL, NULL);
+ xsk_buff_set_rxq_info(rq->umem, &rq->xdp_rxq);
} else {
/* Create a page_pool and register it with rxq */
pp_params.order = 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index d9a5a669b84d..662bc73f6dae 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -263,10 +263,12 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
struct mlx5e_dma_info *dma_info)
{
- if (rq->umem)
- return mlx5e_xsk_page_alloc_umem(rq, dma_info);
- else
- return mlx5e_page_alloc_pool(rq, dma_info);
+ if (rq->umem) {
+ dma_info->xsk = xsk_buff_alloc(rq->umem);
+ return dma_info->xsk ? 0 : -ENOMEM;
+ }
+
+ return mlx5e_page_alloc_pool(rq, dma_info);
}
void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
@@ -300,7 +302,7 @@ static inline void mlx5e_page_release(struct mlx5e_rq *rq,
* put into the Reuse Ring, because there is no way to return
* the page to the userspace when the interface goes down.
*/
- mlx5e_xsk_page_release(rq, dma_info);
+ xsk_buff_free(dma_info->xsk);
else
mlx5e_page_release_dynamic(rq, dma_info, recycle);
}
@@ -385,7 +387,11 @@ static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
if (rq->umem) {
int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
- if (unlikely(!mlx5e_xsk_pages_enough_umem(rq, pages_desired)))
+ /* Check in advance that we have enough frames, instead of
+ * allocating one-by-one, failing and moving frames to the
+ * Reuse Ring.
+ */
+ if (unlikely(!xsk_buff_can_alloc(rq->umem, pages_desired)))
return -ENOMEM;
}
@@ -480,8 +486,11 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
int err;
int i;
+ /* Check in advance that we have enough frames, instead of allocating
+ * one-by-one, failing and moving frames to the Reuse Ring.
+ */
if (rq->umem &&
- unlikely(!mlx5e_xsk_pages_enough_umem(rq, MLX5_MPWRQ_PAGES_PER_WQE))) {
+ unlikely(!xsk_buff_can_alloc(rq->umem, MLX5_MPWRQ_PAGES_PER_WQE))) {
err = -ENOMEM;
goto err;
}
@@ -1038,12 +1047,23 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
return skb;
}
+static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
+ u32 len, struct xdp_buff *xdp)
+{
+ xdp->data_hard_start = va;
+ xdp_set_data_meta_invalid(xdp);
+ xdp->data = va + headroom;
+ xdp->data_end = xdp->data + len;
+ xdp->rxq = &rq->xdp_rxq;
+}
+
struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
{
struct mlx5e_dma_info *di = wi->di;
u16 rx_headroom = rq->buff.headroom;
+ struct xdp_buff xdp;
struct sk_buff *skb;
void *va, *data;
bool consumed;
@@ -1059,11 +1079,13 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
prefetch(data);
rcu_read_lock();
- consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, false);
+ mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
+ consumed = mlx5e_xdp_handle(rq, di, &cqe_bcnt, &xdp);
rcu_read_unlock();
if (consumed)
return NULL; /* page/packet was consumed by XDP */
+ rx_headroom = xdp.data - xdp.data_hard_start;
skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
if (unlikely(!skb))
return NULL;
@@ -1336,6 +1358,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
u16 rx_headroom = rq->buff.headroom;
u32 cqe_bcnt32 = cqe_bcnt;
+ struct xdp_buff xdp;
struct sk_buff *skb;
void *va, *data;
u32 frag_size;
@@ -1357,7 +1380,8 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
prefetch(data);
rcu_read_lock();
- consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, false);
+ mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt32, &xdp);
+ consumed = mlx5e_xdp_handle(rq, di, &cqe_bcnt32, &xdp);
rcu_read_unlock();
if (consumed) {
if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
@@ -1365,6 +1389,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
return NULL; /* page/packet was consumed by XDP */
}
+ rx_headroom = xdp.data - xdp.data_hard_start;
skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32);
if (unlikely(!skb))
return NULL;
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-07 10:42 ` [PATCH bpf-next 10/14] mlx5, " Björn Töpel
@ 2020-05-08 11:55 ` Maxim Mikityanskiy
2020-05-08 12:27 ` Björn Töpel
0 siblings, 1 reply; 24+ messages in thread
From: Maxim Mikityanskiy @ 2020-05-08 11:55 UTC (permalink / raw)
To: Björn Töpel, Björn Töpel
Cc: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher,
maciej.fijalkowski
On 2020-05-07 13:42, Björn Töpel wrote:
> From: Björn Töpel <bjorn.topel@intel.com>
>
> Use the new MEM_TYPE_XSK_BUFF_POOL API in lieu of MEM_TYPE_ZERO_COPY in
> mlx5e. It allows to drop a lot of code from the driver (which is now
> common in AF_XDP core and was related to XSK RX frame allocation, DMA
> mapping, etc.) and slightly improve performance.
>
> rfc->v1: Put back the sanity check for XSK params, use XSK API to get
> the total headroom size. (Maxim)
>
> Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
> Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
I did some functional and performance tests.
Unfortunately, something is wrong with the traffic: I get zeros in
XDP_TX, XDP_PASS and XSK instead of packet data. I set DEBUG_HEXDUMP in
xdpsock, and it shows the packets of the correct length, but all bytes
are 0 after these patches. It might be wrong xdp_buff pointers, however,
I still have to investigate it. Björn, does it also affect Intel
drivers, or is it Mellanox-specific?
For performance, I got +1.0..+1.2 Mpps on RX. TX performance got better
after Björn inlined the relevant UMEM functions, however, there is still
a slight decrease compared to the old code. I'll try to find the
possible reason, but the good thing is that it's not significant anymore.
> ---
> drivers/net/ethernet/mellanox/mlx5/core/en.h | 11 +-
> .../ethernet/mellanox/mlx5/core/en/params.c | 13 +-
> .../net/ethernet/mellanox/mlx5/core/en/xdp.c | 30 ++---
> .../net/ethernet/mellanox/mlx5/core/en/xdp.h | 2 +-
> .../ethernet/mellanox/mlx5/core/en/xsk/rx.c | 111 +++---------------
> .../ethernet/mellanox/mlx5/core/en/xsk/rx.h | 6 -
> .../ethernet/mellanox/mlx5/core/en/xsk/tx.c | 6 +-
> .../ethernet/mellanox/mlx5/core/en/xsk/umem.c | 49 +-------
> .../net/ethernet/mellanox/mlx5/core/en_main.c | 15 +--
> .../net/ethernet/mellanox/mlx5/core/en_rx.c | 43 +++++--
> 10 files changed, 84 insertions(+), 202 deletions(-)
>
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
> index 0864b76ca2c0..368563ff8efb 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
> @@ -426,13 +426,12 @@ struct mlx5e_txqsq {
> } ____cacheline_aligned_in_smp;
>
> struct mlx5e_dma_info {
> - dma_addr_t addr;
> union {
> - struct page *page;
> struct {
> - u64 handle;
> - void *data;
> - } xsk;
> + dma_addr_t addr;
> + struct page *page;
> + };
> + struct xdp_buff *xsk;
> };
> };
>
> @@ -650,7 +649,6 @@ struct mlx5e_rq {
> } mpwqe;
> };
> struct {
> - u16 umem_headroom;
> u16 headroom;
> u8 map_dir; /* dma map direction */
> } buff;
> @@ -682,7 +680,6 @@ struct mlx5e_rq {
> struct page_pool *page_pool;
>
> /* AF_XDP zero-copy */
> - struct zero_copy_allocator zca;
> struct xdp_umem *umem;
>
> struct work_struct recover_work;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
> index eb2e1f2138e4..38e4f19d69f8 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
> @@ -12,15 +12,16 @@ static inline bool mlx5e_rx_is_xdp(struct mlx5e_params *params,
> u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
> struct mlx5e_xsk_param *xsk)
> {
> - u16 headroom = NET_IP_ALIGN;
> + u16 headroom;
>
> - if (mlx5e_rx_is_xdp(params, xsk)) {
> + if (xsk)
> + return xsk->headroom;
> +
> + headroom = NET_IP_ALIGN;
> + if (mlx5e_rx_is_xdp(params, xsk))
> headroom += XDP_PACKET_HEADROOM;
> - if (xsk)
> - headroom += xsk->headroom;
> - } else {
> + else
> headroom += MLX5_RX_HEADROOM;
> - }
>
> return headroom;
> }
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
> index b04b99396f65..a2a194525b15 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
> @@ -71,7 +71,7 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
> xdptxd.data = xdpf->data;
> xdptxd.len = xdpf->len;
>
> - if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) {
> + if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) {
> /* The xdp_buff was in the UMEM and was copied into a newly
> * allocated page. The UMEM page was returned via the ZCA, and
> * this new page has to be mapped at this point and has to be
> @@ -119,49 +119,33 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
>
> /* returns true if packet was consumed by xdp */
> bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
> - void *va, u16 *rx_headroom, u32 *len, bool xsk)
> + u32 *len, struct xdp_buff *xdp)
> {
> struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
> - struct xdp_umem *umem = rq->umem;
> - struct xdp_buff xdp;
> u32 act;
> int err;
>
> if (!prog)
> return false;
>
> - xdp.data = va + *rx_headroom;
> - xdp_set_data_meta_invalid(&xdp);
> - xdp.data_end = xdp.data + *len;
> - xdp.data_hard_start = va;
> - if (xsk)
> - xdp.handle = di->xsk.handle;
> - xdp.rxq = &rq->xdp_rxq;
> -
> - act = bpf_prog_run_xdp(prog, &xdp);
> - if (xsk) {
> - u64 off = xdp.data - xdp.data_hard_start;
> -
> - xdp.handle = xsk_umem_adjust_offset(umem, xdp.handle, off);
> - }
> + act = bpf_prog_run_xdp(prog, xdp);
> switch (act) {
> case XDP_PASS:
> - *rx_headroom = xdp.data - xdp.data_hard_start;
> - *len = xdp.data_end - xdp.data;
> + *len = xdp->data_end - xdp->data;
> return false;
> case XDP_TX:
> - if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, di, &xdp)))
> + if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, di, xdp)))
> goto xdp_abort;
> __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
> return true;
> case XDP_REDIRECT:
> /* When XDP enabled then page-refcnt==1 here */
> - err = xdp_do_redirect(rq->netdev, &xdp, prog);
> + err = xdp_do_redirect(rq->netdev, xdp, prog);
> if (unlikely(err))
> goto xdp_abort;
> __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
> __set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
> - if (!xsk)
> + if (xdp->rxq->mem.type != MEM_TYPE_XSK_BUFF_POOL)
> mlx5e_page_dma_unmap(rq, di);
> rq->stats->xdp_redirect++;
> return true;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
> index ed6f045febeb..54bad625267f 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
> @@ -63,7 +63,7 @@
> struct mlx5e_xsk_param;
> int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
> bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
> - void *va, u16 *rx_headroom, u32 *len, bool xsk);
> + u32 *len, struct xdp_buff *xdp);
> void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
> bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
> void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
> index 62fc8a128a8d..59c40983d428 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
> @@ -3,71 +3,10 @@
>
> #include "rx.h"
> #include "en/xdp.h"
> -#include <net/xdp_sock.h>
> +#include <net/xdp_sock_drv.h>
>
> /* RX data path */
>
> -bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count)
> -{
> - /* Check in advance that we have enough frames, instead of allocating
> - * one-by-one, failing and moving frames to the Reuse Ring.
> - */
> - return xsk_umem_has_addrs_rq(rq->umem, count);
> -}
> -
> -int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
> - struct mlx5e_dma_info *dma_info)
> -{
> - struct xdp_umem *umem = rq->umem;
> - u64 handle;
> -
> - if (!xsk_umem_peek_addr_rq(umem, &handle))
> - return -ENOMEM;
> -
> - dma_info->xsk.handle = xsk_umem_adjust_offset(umem, handle,
> - rq->buff.umem_headroom);
> - dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle);
> -
> - /* No need to add headroom to the DMA address. In striding RQ case, we
> - * just provide pages for UMR, and headroom is counted at the setup
> - * stage when creating a WQE. In non-striding RQ case, headroom is
> - * accounted in mlx5e_alloc_rx_wqe.
> - */
> - dma_info->addr = xdp_umem_get_dma(umem, handle);
> -
> - xsk_umem_release_addr_rq(umem);
> -
> - dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE,
> - DMA_BIDIRECTIONAL);
> -
> - return 0;
> -}
> -
> -static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle)
> -{
> - xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask);
> -}
> -
> -/* XSKRQ uses pages from UMEM, they must not be released. They are returned to
> - * the userspace if possible, and if not, this function is called to reuse them
> - * in the driver.
> - */
> -void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
> - struct mlx5e_dma_info *dma_info)
> -{
> - mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle);
> -}
> -
> -/* Return a frame back to the hardware to fill in again. It is used by XDP when
> - * the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP.
> - */
> -void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
> -{
> - struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca);
> -
> - mlx5e_xsk_recycle_frame(rq, handle);
> -}
> -
> static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data,
> u32 cqe_bcnt)
> {
> @@ -90,11 +29,8 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
> u32 head_offset,
> u32 page_idx)
> {
> - struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
> - u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
> + struct xdp_buff *xdp = wi->umr.dma_info[page_idx].xsk;
> u32 cqe_bcnt32 = cqe_bcnt;
> - void *va, *data;
> - u32 frag_size;
> bool consumed;
>
> /* Check packet size. Note LRO doesn't use linear SKB */
> @@ -103,22 +39,19 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
> return NULL;
> }
>
> - /* head_offset is not used in this function, because di->xsk.data and
> - * di->addr point directly to the necessary place. Furthermore, in the
> - * current implementation, UMR pages are mapped to XSK frames, so
> + /* head_offset is not used in this function, because xdp->data and the
> + * DMA address point directly to the necessary place. Furthermore, in
> + * the current implementation, UMR pages are mapped to XSK frames, so
> * head_offset should always be 0.
> */
> WARN_ON_ONCE(head_offset);
>
> - va = di->xsk.data;
> - data = va + rx_headroom;
> - frag_size = rq->buff.headroom + cqe_bcnt32;
> -
> - dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
> - prefetch(data);
> + xdp->data_end = xdp->data + cqe_bcnt32;
> + xsk_buff_dma_sync_for_cpu(xdp);
> + prefetch(xdp->data);
>
> rcu_read_lock();
> - consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, true);
> + consumed = mlx5e_xdp_handle(rq, NULL, &cqe_bcnt32, xdp);
> rcu_read_unlock();
>
> /* Possible flows:
> @@ -145,7 +78,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
> /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
> * frame. On SKB allocation failure, NULL is returned.
> */
> - return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt32);
> + return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt32);
> }
>
> struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
> @@ -153,25 +86,19 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
> struct mlx5e_wqe_frag_info *wi,
> u32 cqe_bcnt)
> {
> - struct mlx5e_dma_info *di = wi->di;
> - u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
> - void *va, *data;
> + struct xdp_buff *xdp = wi->di->xsk;
> bool consumed;
> - u32 frag_size;
>
> - /* wi->offset is not used in this function, because di->xsk.data and
> - * di->addr point directly to the necessary place. Furthermore, in the
> - * current implementation, one page = one packet = one frame, so
> + /* wi->offset is not used in this function, because xdp->data and the
> + * DMA address point directly to the necessary place. Furthermore, the
> + * XSK allocator allocates frames per packet, instead of pages, so
> * wi->offset should always be 0.
> */
> WARN_ON_ONCE(wi->offset);
>
> - va = di->xsk.data;
> - data = va + rx_headroom;
> - frag_size = rq->buff.headroom + cqe_bcnt;
> -
> - dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
> - prefetch(data);
> + xdp->data_end = xdp->data + cqe_bcnt;
> + xsk_buff_dma_sync_for_cpu(xdp);
> + prefetch(xdp->data);
>
> if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
> rq->stats->wqe_err++;
> @@ -179,7 +106,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
> }
>
> rcu_read_lock();
> - consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, true);
> + consumed = mlx5e_xdp_handle(rq, NULL, &cqe_bcnt, xdp);
> rcu_read_unlock();
>
> if (likely(consumed))
> @@ -189,5 +116,5 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
> * will be handled by mlx5e_put_rx_frag.
> * On SKB allocation failure, NULL is returned.
> */
> - return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt);
> + return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt);
> }
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
> index a8e11adbf426..0062652f37b3 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
> @@ -9,12 +9,6 @@
>
> /* RX data path */
>
> -bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count);
> -int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
> - struct mlx5e_dma_info *dma_info);
> -void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
> - struct mlx5e_dma_info *dma_info);
> -void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle);
> struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
> struct mlx5e_mpw_info *wi,
> u16 cqe_bcnt,
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
> index 3bcdb5b2fc20..470f3b6317c3 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
> @@ -5,7 +5,7 @@
> #include "umem.h"
> #include "en/xdp.h"
> #include "en/params.h"
> -#include <net/xdp_sock.h>
> +#include <net/xdp_sock_drv.h>
>
> int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
> {
> @@ -92,8 +92,8 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
> break;
> }
>
> - xdptxd.dma_addr = xdp_umem_get_dma(umem, desc.addr);
> - xdptxd.data = xdp_umem_get_data(umem, desc.addr);
> + xdptxd.dma_addr = xsk_buff_raw_get_dma(umem, desc.addr);
> + xdptxd.data = xsk_buff_raw_get_data(umem, desc.addr);
> xdptxd.len = desc.len;
>
> dma_sync_single_for_device(sq->pdev, xdptxd.dma_addr,
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
> index 5e49fdb564b3..7b17fcd0a56d 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c
> @@ -10,40 +10,14 @@ static int mlx5e_xsk_map_umem(struct mlx5e_priv *priv,
> struct xdp_umem *umem)
> {
> struct device *dev = priv->mdev->device;
> - u32 i;
>
> - for (i = 0; i < umem->npgs; i++) {
> - dma_addr_t dma = dma_map_page(dev, umem->pgs[i], 0, PAGE_SIZE,
> - DMA_BIDIRECTIONAL);
> -
> - if (unlikely(dma_mapping_error(dev, dma)))
> - goto err_unmap;
> - umem->pages[i].dma = dma;
> - }
> -
> - return 0;
> -
> -err_unmap:
> - while (i--) {
> - dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
> - DMA_BIDIRECTIONAL);
> - umem->pages[i].dma = 0;
> - }
> -
> - return -ENOMEM;
> + return xsk_buff_dma_map(umem, dev, 0);
> }
>
> static void mlx5e_xsk_unmap_umem(struct mlx5e_priv *priv,
> struct xdp_umem *umem)
> {
> - struct device *dev = priv->mdev->device;
> - u32 i;
> -
> - for (i = 0; i < umem->npgs; i++) {
> - dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
> - DMA_BIDIRECTIONAL);
> - umem->pages[i].dma = 0;
> - }
> + return xsk_buff_dma_unmap(umem, 0);
> }
>
> static int mlx5e_xsk_get_umems(struct mlx5e_xsk *xsk)
> @@ -90,13 +64,14 @@ static void mlx5e_xsk_remove_umem(struct mlx5e_xsk *xsk, u16 ix)
>
> static bool mlx5e_xsk_is_umem_sane(struct xdp_umem *umem)
> {
> - return umem->headroom <= 0xffff && umem->chunk_size_nohr <= 0xffff;
> + return xsk_umem_get_headroom(umem) <= 0xffff &&
> + xsk_umem_get_chunk_size(umem) <= 0xffff;
> }
>
> void mlx5e_build_xsk_param(struct xdp_umem *umem, struct mlx5e_xsk_param *xsk)
> {
> - xsk->headroom = umem->headroom;
> - xsk->chunk_size = umem->chunk_size_nohr + umem->headroom;
> + xsk->headroom = xsk_umem_get_headroom(umem);
> + xsk->chunk_size = xsk_umem_get_chunk_size(umem);
> }
>
> static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
> @@ -241,18 +216,6 @@ int mlx5e_xsk_setup_umem(struct net_device *dev, struct xdp_umem *umem, u16 qid)
> mlx5e_xsk_disable_umem(priv, ix);
> }
>
> -int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries)
> -{
> - struct xdp_umem_fq_reuse *reuseq;
> -
> - reuseq = xsk_reuseq_prepare(nentries);
> - if (unlikely(!reuseq))
> - return -ENOMEM;
> - xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
> -
> - return 0;
> -}
> -
> u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk)
> {
> u16 res = xsk->refcnt ? params->num_channels : 0;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
> index 048a4f8601a8..5ce73931eff4 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
> @@ -38,7 +38,7 @@
> #include <linux/bpf.h>
> #include <linux/if_bridge.h>
> #include <net/page_pool.h>
> -#include <net/xdp_sock.h>
> +#include <net/xdp_sock_drv.h>
> #include "eswitch.h"
> #include "en.h"
> #include "en/txrx.h"
> @@ -414,7 +414,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
>
> rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
> rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
> - rq->buff.umem_headroom = xsk ? xsk->headroom : 0;
> pool_size = 1 << params->log_rq_mtu_frames;
>
> switch (rq->wq_type) {
> @@ -522,17 +521,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
> }
>
> if (xsk) {
> - err = mlx5e_xsk_resize_reuseq(umem, num_xsk_frames);
> - if (unlikely(err)) {
> - mlx5_core_err(mdev, "Unable to allocate the Reuse Ring for %u frames\n",
> - num_xsk_frames);
> - goto err_free;
> - }
> -
> - rq->zca.free = mlx5e_xsk_zca_free;
> err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
> - MEM_TYPE_ZERO_COPY,
> - &rq->zca);
> + MEM_TYPE_XSK_BUFF_POOL, NULL);
> + xsk_buff_set_rxq_info(rq->umem, &rq->xdp_rxq);
> } else {
> /* Create a page_pool and register it with rxq */
> pp_params.order = 0;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
> index d9a5a669b84d..662bc73f6dae 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
> @@ -263,10 +263,12 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
> static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
> struct mlx5e_dma_info *dma_info)
> {
> - if (rq->umem)
> - return mlx5e_xsk_page_alloc_umem(rq, dma_info);
> - else
> - return mlx5e_page_alloc_pool(rq, dma_info);
> + if (rq->umem) {
> + dma_info->xsk = xsk_buff_alloc(rq->umem);
> + return dma_info->xsk ? 0 : -ENOMEM;
> + }
> +
> + return mlx5e_page_alloc_pool(rq, dma_info);
> }
>
> void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
> @@ -300,7 +302,7 @@ static inline void mlx5e_page_release(struct mlx5e_rq *rq,
> * put into the Reuse Ring, because there is no way to return
> * the page to the userspace when the interface goes down.
> */
> - mlx5e_xsk_page_release(rq, dma_info);
> + xsk_buff_free(dma_info->xsk);
> else
> mlx5e_page_release_dynamic(rq, dma_info, recycle);
> }
> @@ -385,7 +387,11 @@ static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
> if (rq->umem) {
> int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
>
> - if (unlikely(!mlx5e_xsk_pages_enough_umem(rq, pages_desired)))
> + /* Check in advance that we have enough frames, instead of
> + * allocating one-by-one, failing and moving frames to the
> + * Reuse Ring.
> + */
> + if (unlikely(!xsk_buff_can_alloc(rq->umem, pages_desired)))
> return -ENOMEM;
> }
>
> @@ -480,8 +486,11 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
> int err;
> int i;
>
> + /* Check in advance that we have enough frames, instead of allocating
> + * one-by-one, failing and moving frames to the Reuse Ring.
> + */
> if (rq->umem &&
> - unlikely(!mlx5e_xsk_pages_enough_umem(rq, MLX5_MPWRQ_PAGES_PER_WQE))) {
> + unlikely(!xsk_buff_can_alloc(rq->umem, MLX5_MPWRQ_PAGES_PER_WQE))) {
> err = -ENOMEM;
> goto err;
> }
> @@ -1038,12 +1047,23 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
> return skb;
> }
>
> +static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
> + u32 len, struct xdp_buff *xdp)
> +{
> + xdp->data_hard_start = va;
> + xdp_set_data_meta_invalid(xdp);
> + xdp->data = va + headroom;
> + xdp->data_end = xdp->data + len;
> + xdp->rxq = &rq->xdp_rxq;
> +}
> +
> struct sk_buff *
> mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
> struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
> {
> struct mlx5e_dma_info *di = wi->di;
> u16 rx_headroom = rq->buff.headroom;
> + struct xdp_buff xdp;
> struct sk_buff *skb;
> void *va, *data;
> bool consumed;
> @@ -1059,11 +1079,13 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
> prefetch(data);
>
> rcu_read_lock();
> - consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, false);
> + mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
> + consumed = mlx5e_xdp_handle(rq, di, &cqe_bcnt, &xdp);
> rcu_read_unlock();
> if (consumed)
> return NULL; /* page/packet was consumed by XDP */
>
> + rx_headroom = xdp.data - xdp.data_hard_start;
> skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
> if (unlikely(!skb))
> return NULL;
> @@ -1336,6 +1358,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
> struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
> u16 rx_headroom = rq->buff.headroom;
> u32 cqe_bcnt32 = cqe_bcnt;
> + struct xdp_buff xdp;
> struct sk_buff *skb;
> void *va, *data;
> u32 frag_size;
> @@ -1357,7 +1380,8 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
> prefetch(data);
>
> rcu_read_lock();
> - consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, false);
> + mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt32, &xdp);
> + consumed = mlx5e_xdp_handle(rq, di, &cqe_bcnt32, &xdp);
> rcu_read_unlock();
> if (consumed) {
> if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
> @@ -1365,6 +1389,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
> return NULL; /* page/packet was consumed by XDP */
> }
>
> + rx_headroom = xdp.data - xdp.data_hard_start;
> skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32);
> if (unlikely(!skb))
> return NULL;
>
^ permalink raw reply [flat|nested] 24+ messages in thread* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-08 11:55 ` Maxim Mikityanskiy
@ 2020-05-08 12:27 ` Björn Töpel
2020-05-08 13:00 ` Maxim Mikityanskiy
0 siblings, 1 reply; 24+ messages in thread
From: Björn Töpel @ 2020-05-08 12:27 UTC (permalink / raw)
To: Maxim Mikityanskiy, Björn Töpel
Cc: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher,
maciej.fijalkowski
On 2020-05-08 13:55, Maxim Mikityanskiy wrote:
> On 2020-05-07 13:42, Björn Töpel wrote:
>> From: Björn Töpel <bjorn.topel@intel.com>
>>
>> Use the new MEM_TYPE_XSK_BUFF_POOL API in lieu of MEM_TYPE_ZERO_COPY in
>> mlx5e. It allows to drop a lot of code from the driver (which is now
>> common in AF_XDP core and was related to XSK RX frame allocation, DMA
>> mapping, etc.) and slightly improve performance.
>>
>> rfc->v1: Put back the sanity check for XSK params, use XSK API to get
>> the total headroom size. (Maxim)
>>
>> Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
>> Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
>
> I did some functional and performance tests.
>
> Unfortunately, something is wrong with the traffic: I get zeros in
> XDP_TX, XDP_PASS and XSK instead of packet data. I set DEBUG_HEXDUMP in
> xdpsock, and it shows the packets of the correct length, but all bytes
> are 0 after these patches. It might be wrong xdp_buff pointers, however,
> I still have to investigate it. Björn, does it also affect Intel
> drivers, or is it Mellanox-specific?
>
Are you getting zeros for TX, PASS *and* in xdpsock (REDIRECT:ed
packets), or just TX and PASS?
No, I get correct packet data for AF_XDP zero-copy XDP_REDIRECT,
XDP_PASS, and XDP_TX for Intel.
> For performance, I got +1.0..+1.2 Mpps on RX. TX performance got better
> after Björn inlined the relevant UMEM functions, however, there is still
> a slight decrease compared to the old code. I'll try to find the
> possible reason, but the good thing is that it's not significant anymore.
>
Ok, so for Rx mlx5 it's the same as for i40e. Good! :-)
How much decrease on Tx?
Björn
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-08 12:27 ` Björn Töpel
@ 2020-05-08 13:00 ` Maxim Mikityanskiy
2020-05-08 13:08 ` Björn Töpel
0 siblings, 1 reply; 24+ messages in thread
From: Maxim Mikityanskiy @ 2020-05-08 13:00 UTC (permalink / raw)
To: Björn Töpel, Björn Töpel
Cc: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher,
maciej.fijalkowski
On 2020-05-08 15:27, Björn Töpel wrote:
> On 2020-05-08 13:55, Maxim Mikityanskiy wrote:
>> On 2020-05-07 13:42, Björn Töpel wrote:
>>> From: Björn Töpel <bjorn.topel@intel.com>
>>>
>>> Use the new MEM_TYPE_XSK_BUFF_POOL API in lieu of MEM_TYPE_ZERO_COPY in
>>> mlx5e. It allows to drop a lot of code from the driver (which is now
>>> common in AF_XDP core and was related to XSK RX frame allocation, DMA
>>> mapping, etc.) and slightly improve performance.
>>>
>>> rfc->v1: Put back the sanity check for XSK params, use XSK API to get
>>> the total headroom size. (Maxim)
>>>
>>> Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
>>> Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
>>
>> I did some functional and performance tests.
>>
>> Unfortunately, something is wrong with the traffic: I get zeros in
>> XDP_TX, XDP_PASS and XSK instead of packet data. I set DEBUG_HEXDUMP
>> in xdpsock, and it shows the packets of the correct length, but all
>> bytes are 0 after these patches. It might be wrong xdp_buff pointers,
>> however, I still have to investigate it. Björn, does it also affect
>> Intel drivers, or is it Mellanox-specific?
>>
>
> Are you getting zeros for TX, PASS *and* in xdpsock (REDIRECT:ed
> packets), or just TX and PASS?
Yes, in all modes: XDP_TX, XDP_PASS and XDP_REDIRECT to XSK (xdpsock).
> No, I get correct packet data for AF_XDP zero-copy XDP_REDIRECT,
> XDP_PASS, and XDP_TX for Intel.
Hmm, weird - with the new API I expected the same behavior on all
drivers. Thanks for the information, I'll know that I need to look in
mlx5 code to find the issue.
>> For performance, I got +1.0..+1.2 Mpps on RX. TX performance got
>> better after Björn inlined the relevant UMEM functions, however, there
>> is still a slight decrease compared to the old code. I'll try to find
>> the possible reason, but the good thing is that it's not significant
>> anymore.
>>
>
> Ok, so for Rx mlx5 it's the same as for i40e. Good! :-)
>
> How much decrease on Tx?
~0.8 Mpps (was 3.1 before you inlined the functions).
>
> Björn
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-08 13:00 ` Maxim Mikityanskiy
@ 2020-05-08 13:08 ` Björn Töpel
2020-05-08 13:14 ` Björn Töpel
0 siblings, 1 reply; 24+ messages in thread
From: Björn Töpel @ 2020-05-08 13:08 UTC (permalink / raw)
To: Maxim Mikityanskiy
Cc: Björn Töpel, Alexei Starovoitov, Daniel Borkmann,
David Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Netdev, bpf, Karlsson, Magnus, Jonathan Lemon,
Jeff Kirsher, Fijalkowski, Maciej
On Fri, 8 May 2020 at 15:01, Maxim Mikityanskiy <maximmi@mellanox.com> wrote:
>
> On 2020-05-08 15:27, Björn Töpel wrote:
> > On 2020-05-08 13:55, Maxim Mikityanskiy wrote:
> >> On 2020-05-07 13:42, Björn Töpel wrote:
> >>> From: Björn Töpel <bjorn.topel@intel.com>
> >>>
> >>> Use the new MEM_TYPE_XSK_BUFF_POOL API in lieu of MEM_TYPE_ZERO_COPY in
> >>> mlx5e. It allows to drop a lot of code from the driver (which is now
> >>> common in AF_XDP core and was related to XSK RX frame allocation, DMA
> >>> mapping, etc.) and slightly improve performance.
> >>>
> >>> rfc->v1: Put back the sanity check for XSK params, use XSK API to get
> >>> the total headroom size. (Maxim)
> >>>
> >>> Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
> >>> Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
> >>
> >> I did some functional and performance tests.
> >>
> >> Unfortunately, something is wrong with the traffic: I get zeros in
> >> XDP_TX, XDP_PASS and XSK instead of packet data. I set DEBUG_HEXDUMP
> >> in xdpsock, and it shows the packets of the correct length, but all
> >> bytes are 0 after these patches. It might be wrong xdp_buff pointers,
> >> however, I still have to investigate it. Björn, does it also affect
> >> Intel drivers, or is it Mellanox-specific?
> >>
> >
> > Are you getting zeros for TX, PASS *and* in xdpsock (REDIRECT:ed
> > packets), or just TX and PASS?
>
> Yes, in all modes: XDP_TX, XDP_PASS and XDP_REDIRECT to XSK (xdpsock).
>
> > No, I get correct packet data for AF_XDP zero-copy XDP_REDIRECT,
> > XDP_PASS, and XDP_TX for Intel.
>
> Hmm, weird - with the new API I expected the same behavior on all
> drivers. Thanks for the information, I'll know that I need to look in
> mlx5 code to find the issue.
>
All zeros hints that you're probably putting in the wrong DMA address somewhere.
> >> For performance, I got +1.0..+1.2 Mpps on RX. TX performance got
> >> better after Björn inlined the relevant UMEM functions, however, there
> >> is still a slight decrease compared to the old code. I'll try to find
> >> the possible reason, but the good thing is that it's not significant
> >> anymore.
> >>
> >
> > Ok, so for Rx mlx5 it's the same as for i40e. Good! :-)
> >
> > How much decrease on Tx?
>
> ~0.8 Mpps (was 3.1 before you inlined the functions).
>
Thanks. Still a bit much. What does perf say?
Björn
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-08 13:08 ` Björn Töpel
@ 2020-05-08 13:14 ` Björn Töpel
2020-05-08 17:34 ` Maxim Mikityanskiy
0 siblings, 1 reply; 24+ messages in thread
From: Björn Töpel @ 2020-05-08 13:14 UTC (permalink / raw)
To: Maxim Mikityanskiy
Cc: Björn Töpel, Alexei Starovoitov, Daniel Borkmann,
David Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Netdev, bpf, Karlsson, Magnus, Jonathan Lemon,
Jeff Kirsher, Fijalkowski, Maciej
On Fri, 8 May 2020 at 15:08, Björn Töpel <bjorn.topel@gmail.com> wrote:
>
> On Fri, 8 May 2020 at 15:01, Maxim Mikityanskiy <maximmi@mellanox.com> wrote:
> >
[]
>
> All zeros hints that you're probably putting in the wrong DMA address somewhere.
>
Hmm, I can't see that you're using xsk_buff_xdp_get_dma() anywhere in
the code. Probably it?
Björn
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-08 13:14 ` Björn Töpel
@ 2020-05-08 17:34 ` Maxim Mikityanskiy
2020-05-08 17:57 ` Björn Töpel
0 siblings, 1 reply; 24+ messages in thread
From: Maxim Mikityanskiy @ 2020-05-08 17:34 UTC (permalink / raw)
To: Björn Töpel
Cc: Björn Töpel, Alexei Starovoitov, Daniel Borkmann,
David Miller, Jakub Kicinski, Jesper Dangaard Brouer,
John Fastabend, Netdev, bpf, Karlsson, Magnus, Jonathan Lemon,
Jeff Kirsher, Fijalkowski, Maciej
On 2020-05-08 16:14, Björn Töpel wrote:
> On Fri, 8 May 2020 at 15:08, Björn Töpel <bjorn.topel@gmail.com> wrote:
>>
>> On Fri, 8 May 2020 at 15:01, Maxim Mikityanskiy <maximmi@mellanox.com> wrote:
>>>
> []
>>
>> All zeros hints that you're probably putting in the wrong DMA address somewhere.
>>
>
> Hmm, I can't see that you're using xsk_buff_xdp_get_dma() anywhere in
> the code. Probably it?
You are right, thanks, it was indeed missing. However, adding it was not
enough, I still get zeros, will continue investigating on Monday.
>
> Björn
>
^ permalink raw reply [flat|nested] 24+ messages in thread
* Re: [PATCH bpf-next 10/14] mlx5, xsk: migrate to new MEM_TYPE_XSK_BUFF_POOL
2020-05-08 17:34 ` Maxim Mikityanskiy
@ 2020-05-08 17:57 ` Björn Töpel
0 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-08 17:57 UTC (permalink / raw)
To: Maxim Mikityanskiy, Björn Töpel
Cc: Alexei Starovoitov, Daniel Borkmann, David Miller,
Jakub Kicinski, Jesper Dangaard Brouer, John Fastabend, Netdev,
bpf, Karlsson, Magnus, Jonathan Lemon, Jeff Kirsher, Fijalkowski,
Maciej
On 2020-05-08 19:34, Maxim Mikityanskiy wrote:
[]
>
> You are right, thanks, it was indeed missing. However, adding it was not
> enough, I still get zeros, will continue investigating on Monday.
>
Thanks Max! Let me know if I can help out.
Have a nice weekend,
Björn
^ permalink raw reply [flat|nested] 24+ messages in thread
* [PATCH bpf-next 11/14] xsk: remove MEM_TYPE_ZERO_COPY and corresponding code
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (9 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 10/14] mlx5, " Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 12/14] xdp: simplify xdp_return_{frame,frame_rx_napi,buff} Björn Töpel
` (2 subsequent siblings)
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
There are no users of MEM_TYPE_ZERO_COPY. Remove all corresponding
code, including the "handle" member of struct xdp_buff.
rfc->v1: Fixed spelling in commit message. (Björn)
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
drivers/net/hyperv/netvsc_bpf.c | 1 -
include/net/xdp.h | 9 +--
include/net/xdp_sock.h | 45 -----------
include/net/xdp_sock_drv.h | 139 --------------------------------
include/trace/events/xdp.h | 1 -
net/core/xdp.c | 42 ++--------
net/xdp/xdp_umem.c | 56 +------------
net/xdp/xsk.c | 48 +----------
net/xdp/xsk_buff_pool.c | 7 ++
net/xdp/xsk_queue.c | 62 --------------
net/xdp/xsk_queue.h | 105 ------------------------
11 files changed, 15 insertions(+), 500 deletions(-)
diff --git a/drivers/net/hyperv/netvsc_bpf.c b/drivers/net/hyperv/netvsc_bpf.c
index b86611041db6..9f78f774041b 100644
--- a/drivers/net/hyperv/netvsc_bpf.c
+++ b/drivers/net/hyperv/netvsc_bpf.c
@@ -49,7 +49,6 @@ u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan,
xdp_set_data_meta_invalid(xdp);
xdp->data_end = xdp->data + len;
xdp->rxq = &nvchan->xdp_rxq;
- xdp->handle = 0;
memcpy(xdp->data, data, len);
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 83173e4d306c..1495ffb7a642 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -37,7 +37,6 @@ enum xdp_mem_type {
MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
MEM_TYPE_PAGE_POOL,
- MEM_TYPE_ZERO_COPY,
MEM_TYPE_XSK_BUFF_POOL,
MEM_TYPE_MAX,
};
@@ -53,10 +52,6 @@ struct xdp_mem_info {
struct page_pool;
-struct zero_copy_allocator {
- void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
-};
-
struct xdp_rxq_info {
struct net_device *dev;
u32 queue_index;
@@ -69,7 +64,6 @@ struct xdp_buff {
void *data_end;
void *data_meta;
void *data_hard_start;
- unsigned long handle;
struct xdp_rxq_info *rxq;
};
@@ -102,8 +96,7 @@ struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
int metasize;
int headroom;
- if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
- xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
+ if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
return xdp_convert_zc_to_xdp_frame(xdp);
/* Assure headroom is available for storing info */
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 6e7265f63c04..96bfc5f5f24e 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -17,26 +17,12 @@ struct net_device;
struct xsk_queue;
struct xdp_buff;
-struct xdp_umem_page {
- void *addr;
- dma_addr_t dma;
-};
-
-struct xdp_umem_fq_reuse {
- u32 nentries;
- u32 length;
- u64 handles[];
-};
-
struct xdp_umem {
struct xsk_queue *fq;
struct xsk_queue *cq;
struct xsk_buff_pool *pool;
- struct xdp_umem_page *pages;
- u64 chunk_mask;
u64 size;
u32 headroom;
- u32 chunk_size_nohr;
u32 chunk_size;
struct user_struct *user;
refcount_t users;
@@ -48,7 +34,6 @@ struct xdp_umem {
u8 flags;
int id;
struct net_device *dev;
- struct xdp_umem_fq_reuse *fq_reuse;
bool zc;
spinlock_t xsk_tx_list_lock;
struct list_head xsk_tx_list;
@@ -109,21 +94,6 @@ static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
return xs;
}
-static inline u64 xsk_umem_extract_addr(u64 addr)
-{
- return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
-}
-
-static inline u64 xsk_umem_extract_offset(u64 addr)
-{
- return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
-}
-
-static inline u64 xsk_umem_add_offset_to_addr(u64 addr)
-{
- return xsk_umem_extract_addr(addr) + xsk_umem_extract_offset(addr);
-}
-
#else
static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
@@ -146,21 +116,6 @@ static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
return NULL;
}
-static inline u64 xsk_umem_extract_addr(u64 addr)
-{
- return 0;
-}
-
-static inline u64 xsk_umem_extract_offset(u64 addr)
-{
- return 0;
-}
-
-static inline u64 xsk_umem_add_offset_to_addr(u64 addr)
-{
- return 0;
-}
-
#endif /* CONFIG_XDP_SOCKETS */
#endif /* _LINUX_XDP_SOCK_H */
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index 5a0970d4c44c..533ee0ce43de 100644
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -11,16 +11,9 @@
#ifdef CONFIG_XDP_SOCKETS
-bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt);
-bool xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr);
-void xsk_umem_release_addr(struct xdp_umem *umem);
void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries);
bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc);
void xsk_umem_consume_tx_done(struct xdp_umem *umem);
-struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries);
-struct xdp_umem_fq_reuse *xsk_reuseq_swap(struct xdp_umem *umem,
- struct xdp_umem_fq_reuse *newq);
-void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq);
struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, u16 queue_id);
void xsk_set_rx_need_wakeup(struct xdp_umem *umem);
void xsk_set_tx_need_wakeup(struct xdp_umem *umem);
@@ -28,75 +21,6 @@ void xsk_clear_rx_need_wakeup(struct xdp_umem *umem);
void xsk_clear_tx_need_wakeup(struct xdp_umem *umem);
bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem);
-static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)
-{
- unsigned long page_addr;
-
- addr = xsk_umem_add_offset_to_addr(addr);
- page_addr = (unsigned long)umem->pages[addr >> PAGE_SHIFT].addr;
-
- return (char *)(page_addr & PAGE_MASK) + (addr & ~PAGE_MASK);
-}
-
-static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr)
-{
- addr = xsk_umem_add_offset_to_addr(addr);
-
- return umem->pages[addr >> PAGE_SHIFT].dma + (addr & ~PAGE_MASK);
-}
-
-/* Reuse-queue aware version of FILL queue helpers */
-static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- if (rq->length >= cnt)
- return true;
-
- return xsk_umem_has_addrs(umem, cnt - rq->length);
-}
-
-static inline bool xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- if (!rq->length)
- return xsk_umem_peek_addr(umem, addr);
-
- *addr = rq->handles[rq->length - 1];
- return addr;
-}
-
-static inline void xsk_umem_release_addr_rq(struct xdp_umem *umem)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- if (!rq->length)
- xsk_umem_release_addr(umem);
- else
- rq->length--;
-}
-
-static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr)
-{
- struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
-
- rq->handles[rq->length++] = addr;
-}
-
-/* Handle the offset appropriately depending on aligned or unaligned mode.
- * For unaligned mode, we store the offset in the upper 16-bits of the address.
- * For aligned mode, we simply add the offset to the address.
- */
-static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 address,
- u64 offset)
-{
- if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG)
- return address + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
- else
- return address + offset;
-}
-
static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
{
return XDP_PACKET_HEADROOM + umem->headroom;
@@ -180,20 +104,6 @@ static inline void xsk_buff_raw_dma_sync_for_device(struct xdp_umem *umem,
#else
-static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
-{
- return false;
-}
-
-static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
-{
- return NULL;
-}
-
-static inline void xsk_umem_release_addr(struct xdp_umem *umem)
-{
-}
-
static inline void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries)
{
}
@@ -208,55 +118,12 @@ static inline void xsk_umem_consume_tx_done(struct xdp_umem *umem)
{
}
-static inline struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries)
-{
- return NULL;
-}
-
-static inline struct xdp_umem_fq_reuse *xsk_reuseq_swap(
- struct xdp_umem *umem, struct xdp_umem_fq_reuse *newq)
-{
- return NULL;
-}
-
-static inline void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq)
-{
-}
-
static inline struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev,
u16 queue_id)
{
return NULL;
}
-static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr)
-{
- return NULL;
-}
-
-static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr)
-{
- return 0;
-}
-
-static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
-{
- return false;
-}
-
-static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
-{
- return NULL;
-}
-
-static inline void xsk_umem_release_addr_rq(struct xdp_umem *umem)
-{
-}
-
-static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr)
-{
-}
-
static inline void xsk_set_rx_need_wakeup(struct xdp_umem *umem)
{
}
@@ -278,12 +145,6 @@ static inline bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
return false;
}
-static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 handle,
- u64 offset)
-{
- return 0;
-}
-
static inline u32 xsk_umem_get_headroom(struct xdp_umem *umem)
{
return 0;
diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index 48547a12fa27..b73d3e141323 100644
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -287,7 +287,6 @@ TRACE_EVENT(xdp_devmap_xmit,
FN(PAGE_SHARED) \
FN(PAGE_ORDER0) \
FN(PAGE_POOL) \
- FN(ZERO_COPY) \
FN(XSK_BUFF_POOL)
#define __MEM_TYPE_TP_FN(x) \
diff --git a/net/core/xdp.c b/net/core/xdp.c
index 89053ef8333b..11273c976e19 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -109,27 +109,6 @@ static void mem_allocator_disconnect(void *allocator)
mutex_unlock(&mem_id_lock);
}
-static void mem_id_disconnect(int id)
-{
- struct xdp_mem_allocator *xa;
-
- mutex_lock(&mem_id_lock);
-
- xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
- if (!xa) {
- mutex_unlock(&mem_id_lock);
- WARN(1, "Request remove non-existing id(%d), driver bug?", id);
- return;
- }
-
- trace_mem_disconnect(xa);
-
- if (!rhashtable_remove_fast(mem_id_ht, &xa->node, mem_id_rht_params))
- call_rcu(&xa->rcu, __xdp_mem_allocator_rcu_free);
-
- mutex_unlock(&mem_id_lock);
-}
-
void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq)
{
struct xdp_mem_allocator *xa;
@@ -143,9 +122,6 @@ void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq)
if (id == 0)
return;
- if (xdp_rxq->mem.type == MEM_TYPE_ZERO_COPY)
- return mem_id_disconnect(id);
-
if (xdp_rxq->mem.type == MEM_TYPE_PAGE_POOL) {
rcu_read_lock();
xa = rhashtable_lookup(mem_id_ht, &id, mem_id_rht_params);
@@ -301,7 +277,7 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
xdp_rxq->mem.type = type;
if (!allocator) {
- if (type == MEM_TYPE_PAGE_POOL || type == MEM_TYPE_ZERO_COPY)
+ if (type == MEM_TYPE_PAGE_POOL)
return -EINVAL; /* Setup time check page_pool req */
return 0;
}
@@ -361,7 +337,7 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
* of xdp_frames/pages in those cases.
*/
static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
- unsigned long handle, struct xdp_buff *xdp)
+ struct xdp_buff *xdp)
{
struct xdp_mem_allocator *xa;
struct page *page;
@@ -383,14 +359,6 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
page = virt_to_page(data); /* Assumes order0 page*/
put_page(page);
break;
- case MEM_TYPE_ZERO_COPY:
- /* NB! Only valid from an xdp_buff! */
- rcu_read_lock();
- /* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
- xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
- xa->zc_alloc->free(xa->zc_alloc, handle);
- rcu_read_unlock();
- break;
case MEM_TYPE_XSK_BUFF_POOL:
/* NB! Only valid from an xdp_buff! */
xsk_buff_free(xdp);
@@ -403,19 +371,19 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
void xdp_return_frame(struct xdp_frame *xdpf)
{
- __xdp_return(xdpf->data, &xdpf->mem, false, 0, NULL);
+ __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
}
EXPORT_SYMBOL_GPL(xdp_return_frame);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
{
- __xdp_return(xdpf->data, &xdpf->mem, true, 0, NULL);
+ __xdp_return(xdpf->data, &xdpf->mem, true, NULL);
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
void xdp_return_buff(struct xdp_buff *xdp)
{
- __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle, xdp);
+ __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);
}
EXPORT_SYMBOL_GPL(xdp_return_buff);
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 7f04688045d5..19e59d1a5e9f 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -179,37 +179,6 @@ void xdp_umem_clear_dev(struct xdp_umem *umem)
umem->zc = false;
}
-static void xdp_umem_unmap_pages(struct xdp_umem *umem)
-{
- unsigned int i;
-
- for (i = 0; i < umem->npgs; i++)
- if (PageHighMem(umem->pgs[i]))
- vunmap(umem->pages[i].addr);
-}
-
-static int xdp_umem_map_pages(struct xdp_umem *umem)
-{
- unsigned int i;
- void *addr;
-
- for (i = 0; i < umem->npgs; i++) {
- if (PageHighMem(umem->pgs[i]))
- addr = vmap(&umem->pgs[i], 1, VM_MAP, PAGE_KERNEL);
- else
- addr = page_address(umem->pgs[i]);
-
- if (!addr) {
- xdp_umem_unmap_pages(umem);
- return -ENOMEM;
- }
-
- umem->pages[i].addr = addr;
- }
-
- return 0;
-}
-
static void xdp_umem_unpin_pages(struct xdp_umem *umem)
{
unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
@@ -244,14 +213,9 @@ static void xdp_umem_release(struct xdp_umem *umem)
umem->cq = NULL;
}
- xsk_reuseq_destroy(umem);
xp_destroy(umem->pool);
- xdp_umem_unmap_pages(umem);
xdp_umem_unpin_pages(umem);
- kvfree(umem->pages);
- umem->pages = NULL;
-
xdp_umem_unaccount_pages(umem);
kfree(umem);
}
@@ -385,11 +349,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
return -EINVAL;
- umem->chunk_mask = unaligned_chunks ? XSK_UNALIGNED_BUF_ADDR_MASK
- : ~((u64)chunk_size - 1);
umem->size = size;
umem->headroom = headroom;
- umem->chunk_size_nohr = chunk_size - headroom;
umem->chunk_size = chunk_size;
umem->npgs = size / PAGE_SIZE;
umem->pgs = NULL;
@@ -408,29 +369,14 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
if (err)
goto out_account;
- umem->pages = kvcalloc(umem->npgs, sizeof(*umem->pages),
- GFP_KERNEL_ACCOUNT);
- if (!umem->pages) {
- err = -ENOMEM;
- goto out_pin;
- }
-
- err = xdp_umem_map_pages(umem);
- if (err)
- goto out_pages;
-
umem->pool = xp_create(umem->pgs, umem->npgs, chunks, chunk_size,
headroom, size, unaligned_chunks);
if (!umem->pool) {
err = -ENOMEM;
- goto out_unmap;
+ goto out_pin;
}
return 0;
-out_unmap:
- xdp_umem_unmap_pages(umem);
-out_pages:
- kvfree(umem->pages);
out_pin:
xdp_umem_unpin_pages(umem);
out_account:
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 6933f0d494ba..3f2ab732ab8b 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -39,24 +39,6 @@ bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
READ_ONCE(xs->umem->fq);
}
-bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
-{
- return xskq_cons_has_entries(umem->fq, cnt);
-}
-EXPORT_SYMBOL(xsk_umem_has_addrs);
-
-bool xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
-{
- return xskq_cons_peek_addr(umem->fq, addr, umem);
-}
-EXPORT_SYMBOL(xsk_umem_peek_addr);
-
-void xsk_umem_release_addr(struct xdp_umem *umem)
-{
- xskq_cons_release(umem->fq);
-}
-EXPORT_SYMBOL(xsk_umem_release_addr);
-
void xsk_set_rx_need_wakeup(struct xdp_umem *umem)
{
if (umem->need_wakeup & XDP_WAKEUP_RX)
@@ -203,8 +185,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,
len = xdp->data_end - xdp->data;
- return xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY ||
- xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
+ return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?
__xsk_rcv_zc(xs, xdp, len) :
__xsk_rcv(xs, xdp, len, explicit_free);
}
@@ -588,24 +569,6 @@ static struct socket *xsk_lookup_xsk_from_fd(int fd)
return sock;
}
-/* Check if umem pages are contiguous.
- * If zero-copy mode, use the DMA address to do the page contiguity check
- * For all other modes we use addr (kernel virtual address)
- * Store the result in the low bits of addr.
- */
-static void xsk_check_page_contiguity(struct xdp_umem *umem, u32 flags)
-{
- struct xdp_umem_page *pgs = umem->pages;
- int i, is_contig;
-
- for (i = 0; i < umem->npgs - 1; i++) {
- is_contig = (flags & XDP_ZEROCOPY) ?
- (pgs[i].dma + PAGE_SIZE == pgs[i + 1].dma) :
- (pgs[i].addr + PAGE_SIZE == pgs[i + 1].addr);
- pgs[i].addr += is_contig << XSK_NEXT_PG_CONTIG_SHIFT;
- }
-}
-
static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
{
struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr;
@@ -688,23 +651,14 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
goto out_unlock;
} else {
/* This xsk has its own umem. */
- xskq_set_umem(xs->umem->fq, xs->umem->size,
- xs->umem->chunk_mask);
- xskq_set_umem(xs->umem->cq, xs->umem->size,
- xs->umem->chunk_mask);
-
err = xdp_umem_assign_dev(xs->umem, dev, qid, flags);
if (err)
goto out_unlock;
-
- xsk_check_page_contiguity(xs->umem, flags);
}
xs->dev = dev;
xs->zc = xs->umem->zc;
xs->queue_id = qid;
- xskq_set_umem(xs->rx, xs->umem->size, xs->umem->chunk_mask);
- xskq_set_umem(xs->tx, xs->umem->size, xs->umem->chunk_mask);
xdp_add_sk_umem(xs->umem, xs);
out_unlock:
diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index df5db2c38859..365bdb5749cc 100644
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -8,6 +8,13 @@
#include "xsk_queue.h"
+/* Masks for xdp_umem_page flags.
+ * The low 12-bits of the addr will be 0 since this is the page address, so we
+ * can use them for flags.
+ */
+#define XSK_NEXT_PG_CONTIG_SHIFT 0
+#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT)
+
struct xsk_buff_pool {
struct xsk_queue *fq;
struct list_head free_list;
diff --git a/net/xdp/xsk_queue.c b/net/xdp/xsk_queue.c
index 57fb81bd593c..c33b5d985b9b 100644
--- a/net/xdp/xsk_queue.c
+++ b/net/xdp/xsk_queue.c
@@ -9,15 +9,6 @@
#include "xsk_queue.h"
-void xskq_set_umem(struct xsk_queue *q, u64 umem_size, u64 chunk_mask)
-{
- if (!q)
- return;
-
- q->umem_size = umem_size;
- q->chunk_mask = chunk_mask;
-}
-
static size_t xskq_get_ring_size(struct xsk_queue *q, bool umem_queue)
{
struct xdp_umem_ring *umem_ring;
@@ -63,56 +54,3 @@ void xskq_destroy(struct xsk_queue *q)
page_frag_free(q->ring);
kfree(q);
}
-
-struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries)
-{
- struct xdp_umem_fq_reuse *newq;
-
- /* Check for overflow */
- if (nentries > (u32)roundup_pow_of_two(nentries))
- return NULL;
- nentries = roundup_pow_of_two(nentries);
-
- newq = kvmalloc(struct_size(newq, handles, nentries), GFP_KERNEL);
- if (!newq)
- return NULL;
- memset(newq, 0, offsetof(typeof(*newq), handles));
-
- newq->nentries = nentries;
- return newq;
-}
-EXPORT_SYMBOL_GPL(xsk_reuseq_prepare);
-
-struct xdp_umem_fq_reuse *xsk_reuseq_swap(struct xdp_umem *umem,
- struct xdp_umem_fq_reuse *newq)
-{
- struct xdp_umem_fq_reuse *oldq = umem->fq_reuse;
-
- if (!oldq) {
- umem->fq_reuse = newq;
- return NULL;
- }
-
- if (newq->nentries < oldq->length)
- return newq;
-
- memcpy(newq->handles, oldq->handles,
- array_size(oldq->length, sizeof(u64)));
- newq->length = oldq->length;
-
- umem->fq_reuse = newq;
- return oldq;
-}
-EXPORT_SYMBOL_GPL(xsk_reuseq_swap);
-
-void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq)
-{
- kvfree(rq);
-}
-EXPORT_SYMBOL_GPL(xsk_reuseq_free);
-
-void xsk_reuseq_destroy(struct xdp_umem *umem)
-{
- xsk_reuseq_free(umem->fq_reuse);
- umem->fq_reuse = NULL;
-}
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index 9151aef7dbca..16bf15864788 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -32,8 +32,6 @@ struct xdp_umem_ring {
};
struct xsk_queue {
- u64 chunk_mask;
- u64 umem_size;
u32 ring_mask;
u32 nentries;
u32 cached_prod;
@@ -106,90 +104,6 @@ struct xsk_queue {
/* Functions that read and validate content from consumer rings. */
-static inline bool xskq_cons_crosses_non_contig_pg(struct xdp_umem *umem,
- u64 addr,
- u64 length)
-{
- bool cross_pg = (addr & (PAGE_SIZE - 1)) + length > PAGE_SIZE;
- bool next_pg_contig =
- (unsigned long)umem->pages[(addr >> PAGE_SHIFT)].addr &
- XSK_NEXT_PG_CONTIG_MASK;
-
- return cross_pg && !next_pg_contig;
-}
-
-static inline bool xskq_cons_is_valid_unaligned(struct xsk_queue *q,
- u64 addr,
- u64 length,
- struct xdp_umem *umem)
-{
- u64 base_addr = xsk_umem_extract_addr(addr);
-
- addr = xsk_umem_add_offset_to_addr(addr);
- if (base_addr >= q->umem_size || addr >= q->umem_size ||
- xskq_cons_crosses_non_contig_pg(umem, addr, length)) {
- q->invalid_descs++;
- return false;
- }
-
- return true;
-}
-
-static inline bool xskq_cons_is_valid_addr(struct xsk_queue *q, u64 addr)
-{
- if (addr >= q->umem_size) {
- q->invalid_descs++;
- return false;
- }
-
- return true;
-}
-
-static inline bool xskq_cons_read_addr(struct xsk_queue *q, u64 *addr,
- struct xdp_umem *umem)
-{
- struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
-
- while (q->cached_cons != q->cached_prod) {
- u32 idx = q->cached_cons & q->ring_mask;
-
- *addr = ring->desc[idx] & q->chunk_mask;
-
- if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG) {
- if (xskq_cons_is_valid_unaligned(q, *addr,
- umem->chunk_size_nohr,
- umem))
- return true;
- goto out;
- }
-
- if (xskq_cons_is_valid_addr(q, *addr))
- return true;
-
-out:
- q->cached_cons++;
- }
-
- return false;
-}
-
-static inline bool xskq_cons_read_addr_aligned(struct xsk_queue *q, u64 *addr)
-{
- struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
-
- while (q->cached_cons != q->cached_prod) {
- u32 idx = q->cached_cons & q->ring_mask;
-
- *addr = ring->desc[idx];
- if (xskq_cons_is_valid_addr(q, *addr))
- return true;
-
- q->cached_cons++;
- }
-
- return false;
-}
-
static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr)
{
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
@@ -267,21 +181,6 @@ static inline bool xskq_cons_has_entries(struct xsk_queue *q, u32 cnt)
return entries >= cnt;
}
-static inline bool xskq_cons_peek_addr(struct xsk_queue *q, u64 *addr,
- struct xdp_umem *umem)
-{
- if (q->cached_prod == q->cached_cons)
- xskq_cons_get_entries(q);
- return xskq_cons_read_addr(q, addr, umem);
-}
-
-static inline bool xskq_cons_peek_addr_aligned(struct xsk_queue *q, u64 *addr)
-{
- if (q->cached_prod == q->cached_cons)
- xskq_cons_get_entries(q);
- return xskq_cons_read_addr_aligned(q, addr);
-}
-
static inline bool xskq_cons_peek_addr_unchecked(struct xsk_queue *q, u64 *addr)
{
if (q->cached_prod == q->cached_cons)
@@ -410,11 +309,7 @@ static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
return q ? q->invalid_descs : 0;
}
-void xskq_set_umem(struct xsk_queue *q, u64 umem_size, u64 chunk_mask);
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
void xskq_destroy(struct xsk_queue *q_ops);
-/* Executed by the core when the entire UMEM gets freed */
-void xsk_reuseq_destroy(struct xdp_umem *umem);
-
#endif /* _LINUX_XSK_QUEUE_H */
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 12/14] xdp: simplify xdp_return_{frame,frame_rx_napi,buff}
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (10 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 11/14] xsk: remove MEM_TYPE_ZERO_COPY and corresponding code Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 13/14] xsk: explicitly inline functions and move definitions Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 14/14] MAINTAINERS, xsk: update AF_XDP section after moves/adds Björn Töpel
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
The xdp_return_{frame,frame_rx_napi,buff} function are never used,
except in xdp_convert_zc_to_xdp_frame(), by the MEM_TYPE_XSK_BUFF_POOL
memory type.
To simplify and reduce code, change so that
xdp_convert_zc_to_xdp_frame() calls xsk_buff_free() directly since the
type is know, and remove MEM_TYPE_XSK_BUFF_POOL from the switch
statement in __xdp_return() function.
Suggested-by: Maxim Mikityanskiy <maximmi@mellanox.com>
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
net/core/xdp.c | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
diff --git a/net/core/xdp.c b/net/core/xdp.c
index 11273c976e19..7ab1f9014c5e 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -334,10 +334,11 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
* scenarios (e.g. queue full), it is possible to return the xdp_frame
* while still leveraging this protection. The @napi_direct boolean
* is used for those calls sites. Thus, allowing for faster recycling
- * of xdp_frames/pages in those cases.
+ * of xdp_frames/pages in those cases. This path is never used by the
+ * MEM_TYPE_XSK_BUFF_POOL memory type, so it's explicitly not part of
+ * the switch-statement.
*/
-static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
- struct xdp_buff *xdp)
+static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct)
{
struct xdp_mem_allocator *xa;
struct page *page;
@@ -359,33 +360,29 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
page = virt_to_page(data); /* Assumes order0 page*/
put_page(page);
break;
- case MEM_TYPE_XSK_BUFF_POOL:
- /* NB! Only valid from an xdp_buff! */
- xsk_buff_free(xdp);
- break;
default:
/* Not possible, checked in xdp_rxq_info_reg_mem_model() */
+ WARN(1, "Incorrect XDP memory type (%d) usage", mem->type);
break;
}
}
void xdp_return_frame(struct xdp_frame *xdpf)
{
- __xdp_return(xdpf->data, &xdpf->mem, false, NULL);
+ __xdp_return(xdpf->data, &xdpf->mem, false);
}
EXPORT_SYMBOL_GPL(xdp_return_frame);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
{
- __xdp_return(xdpf->data, &xdpf->mem, true, NULL);
+ __xdp_return(xdpf->data, &xdpf->mem, true);
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
void xdp_return_buff(struct xdp_buff *xdp)
{
- __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);
+ __xdp_return(xdp->data, &xdp->rxq->mem, true);
}
-EXPORT_SYMBOL_GPL(xdp_return_buff);
/* Only called for MEM_TYPE_PAGE_POOL see xdp.h */
void __xdp_release_frame(void *data, struct xdp_mem_info *mem)
@@ -466,7 +463,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
xdpf->metasize = metasize;
xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
- xdp_return_buff(xdp);
+ xsk_buff_free(xdp);
return xdpf;
}
EXPORT_SYMBOL_GPL(xdp_convert_zc_to_xdp_frame);
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 13/14] xsk: explicitly inline functions and move definitions
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (11 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 12/14] xdp: simplify xdp_return_{frame,frame_rx_napi,buff} Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
2020-05-07 10:42 ` [PATCH bpf-next 14/14] MAINTAINERS, xsk: update AF_XDP section after moves/adds Björn Töpel
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski
From: Björn Töpel <bjorn.topel@intel.com>
In order to reduce the number of function calls, the struct
xsk_buff_pool definition is moved to xsk_buff_pool.h. The functions
xp_get_dma(), xp_dma_sync_for_cpu(), xp_dma_sync_for_device(),
xp_validate_desc() and various helper functions are explicitly
inlined.
Further, move xp_get_handle() and xp_release() to xsk.c, to allow for
the compiler to perform inlining.
rfc->v1: Make sure xp_validate_desc() is inlined for Tx perf. (Maxim)
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
include/net/xsk_buff_pool.h | 92 +++++++++++++++++++++--
net/xdp/xsk.c | 15 ++++
net/xdp/xsk_buff_pool.c | 142 ++----------------------------------
net/xdp/xsk_queue.h | 45 ++++++++++++
4 files changed, 151 insertions(+), 143 deletions(-)
diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
index 9abef166441d..029522696ccb 100644
--- a/include/net/xsk_buff_pool.h
+++ b/include/net/xsk_buff_pool.h
@@ -4,6 +4,7 @@
#ifndef XSK_BUFF_POOL_H_
#define XSK_BUFF_POOL_H_
+#include <linux/if_xdp.h>
#include <linux/types.h>
#include <linux/dma-mapping.h>
#include <net/xdp.h>
@@ -24,6 +25,27 @@ struct xdp_buff_xsk {
struct list_head free_list_node;
};
+struct xsk_buff_pool {
+ struct xsk_queue *fq;
+ struct list_head free_list;
+ dma_addr_t *dma_pages;
+ struct xdp_buff_xsk *heads;
+ u64 chunk_mask;
+ u64 addrs_cnt;
+ u32 free_list_cnt;
+ u32 dma_pages_cnt;
+ u32 heads_cnt;
+ u32 free_heads_cnt;
+ u32 headroom;
+ u32 chunk_size;
+ u32 frame_len;
+ bool cheap_dma;
+ bool unaligned;
+ void *addrs;
+ struct device *dev;
+ struct xdp_buff_xsk *free_heads[];
+};
+
/* AF_XDP core. */
struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
u32 chunk_size, u32 headroom, u64 size,
@@ -31,8 +53,6 @@ struct xsk_buff_pool *xp_create(struct page **pages, u32 nr_pages, u32 chunks,
void xp_set_fq(struct xsk_buff_pool *pool, struct xsk_queue *fq);
void xp_destroy(struct xsk_buff_pool *pool);
void xp_release(struct xdp_buff_xsk *xskb);
-u64 xp_get_handle(struct xdp_buff_xsk *xskb);
-bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
/* AF_XDP, and XDP core. */
void xp_free(struct xdp_buff_xsk *xskb);
@@ -46,9 +66,69 @@ struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
-dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb);
-void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb);
-void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
- size_t size);
+static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb)
+{
+ return xskb->dma;
+}
+
+void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb);
+static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb)
+{
+ if (xskb->pool->cheap_dma)
+ return;
+
+ xp_dma_sync_for_cpu_slow(xskb);
+}
+
+void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
+ size_t size);
+static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool,
+ dma_addr_t dma, size_t size)
+{
+ if (pool->cheap_dma)
+ return;
+
+ xp_dma_sync_for_device_slow(pool, dma, size);
+}
+
+/* Masks for xdp_umem_page flags.
+ * The low 12-bits of the addr will be 0 since this is the page address, so we
+ * can use them for flags.
+ */
+#define XSK_NEXT_PG_CONTIG_SHIFT 0
+#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT)
+
+static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,
+ u64 addr, u32 len)
+{
+ bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE;
+
+ if (pool->dma_pages_cnt && cross_pg) {
+ return !(pool->dma_pages[addr >> PAGE_SHIFT] &
+ XSK_NEXT_PG_CONTIG_MASK);
+ }
+ return false;
+}
+
+static inline u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)
+{
+ return addr & pool->chunk_mask;
+}
+
+static inline u64 xp_unaligned_extract_addr(u64 addr)
+{
+ return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
+}
+
+static inline u64 xp_unaligned_extract_offset(u64 addr)
+{
+ return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
+}
+
+static inline u64 xp_unaligned_add_offset_to_addr(u64 addr)
+{
+ return xp_unaligned_extract_addr(addr) +
+ xp_unaligned_extract_offset(addr);
+}
#endif /* XSK_BUFF_POOL_H_ */
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 3f2ab732ab8b..b6c0f08bd80d 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -99,6 +99,21 @@ bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem)
}
EXPORT_SYMBOL(xsk_umem_uses_need_wakeup);
+void xp_release(struct xdp_buff_xsk *xskb)
+{
+ xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;
+}
+
+static u64 xp_get_handle(struct xdp_buff_xsk *xskb)
+{
+ u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;
+
+ offset += xskb->pool->headroom;
+ if (!xskb->pool->unaligned)
+ return xskb->orig_addr + offset;
+ return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
+}
+
static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index 365bdb5749cc..20ea583133de 100644
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -8,34 +8,6 @@
#include "xsk_queue.h"
-/* Masks for xdp_umem_page flags.
- * The low 12-bits of the addr will be 0 since this is the page address, so we
- * can use them for flags.
- */
-#define XSK_NEXT_PG_CONTIG_SHIFT 0
-#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT)
-
-struct xsk_buff_pool {
- struct xsk_queue *fq;
- struct list_head free_list;
- dma_addr_t *dma_pages;
- struct xdp_buff_xsk *heads;
- u64 chunk_mask;
- u64 addrs_cnt;
- u32 free_list_cnt;
- u32 dma_pages_cnt;
- u32 heads_cnt;
- u32 free_heads_cnt;
- u32 headroom;
- u32 chunk_size;
- u32 frame_len;
- bool cheap_dma;
- bool unaligned;
- void *addrs;
- struct device *dev;
- struct xdp_buff_xsk *free_heads[];
-};
-
static void xp_addr_unmap(struct xsk_buff_pool *pool)
{
vunmap(pool->addrs);
@@ -227,50 +199,12 @@ int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
}
EXPORT_SYMBOL(xp_dma_map);
-static bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,
- u64 addr, u32 len)
-{
- bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE;
-
- if (pool->dma_pages_cnt && cross_pg) {
- return !(pool->dma_pages[addr >> PAGE_SHIFT] &
- XSK_NEXT_PG_CONTIG_MASK);
- }
- return false;
-}
-
static bool xp_addr_crosses_non_contig_pg(struct xsk_buff_pool *pool,
u64 addr)
{
return xp_desc_crosses_non_contig_pg(pool, addr, pool->chunk_size);
}
-void xp_release(struct xdp_buff_xsk *xskb)
-{
- xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;
-}
-
-static u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)
-{
- return addr & pool->chunk_mask;
-}
-
-static u64 xp_unaligned_extract_addr(u64 addr)
-{
- return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
-}
-
-static u64 xp_unaligned_extract_offset(u64 addr)
-{
- return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
-}
-
-static u64 xp_unaligned_add_offset_to_addr(u64 addr)
-{
- return xp_unaligned_extract_addr(addr) +
- xp_unaligned_extract_offset(addr);
-}
-
static bool xp_check_unaligned(struct xsk_buff_pool *pool, u64 *addr)
{
*addr = xp_unaligned_extract_addr(*addr);
@@ -368,60 +302,6 @@ void xp_free(struct xdp_buff_xsk *xskb)
}
EXPORT_SYMBOL(xp_free);
-static bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
- struct xdp_desc *desc)
-{
- u64 chunk, chunk_end;
-
- chunk = xp_aligned_extract_addr(pool, desc->addr);
- chunk_end = xp_aligned_extract_addr(pool, desc->addr + desc->len);
- if (chunk != chunk_end)
- return false;
-
- if (chunk >= pool->addrs_cnt)
- return false;
-
- if (desc->options)
- return false;
- return true;
-}
-
-static bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool,
- struct xdp_desc *desc)
-{
- u64 addr, base_addr;
-
- base_addr = xp_unaligned_extract_addr(desc->addr);
- addr = xp_unaligned_add_offset_to_addr(desc->addr);
-
- if (desc->len > pool->chunk_size)
- return false;
-
- if (base_addr >= pool->addrs_cnt || addr >= pool->addrs_cnt ||
- xp_desc_crosses_non_contig_pg(pool, addr, desc->len))
- return false;
-
- if (desc->options)
- return false;
- return true;
-}
-
-bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
-{
- return pool->unaligned ? xp_unaligned_validate_desc(pool, desc) :
- xp_aligned_validate_desc(pool, desc);
-}
-
-u64 xp_get_handle(struct xdp_buff_xsk *xskb)
-{
- u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;
-
- offset += xskb->pool->headroom;
- if (!xskb->pool->unaligned)
- return xskb->orig_addr + offset;
- return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
-}
-
void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
{
addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
@@ -438,32 +318,20 @@ dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr)
}
EXPORT_SYMBOL(xp_raw_get_dma);
-dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb)
-{
- return xskb->dma;
-}
-EXPORT_SYMBOL(xp_get_dma);
-
-void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb)
+void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb)
{
size_t size;
- if (xskb->pool->cheap_dma)
- return;
-
size = xskb->xdp.data_end - xskb->xdp.data;
dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0,
size, DMA_BIDIRECTIONAL);
}
-EXPORT_SYMBOL(xp_dma_sync_for_cpu);
+EXPORT_SYMBOL(xp_dma_sync_for_cpu_slow);
-void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma,
- size_t size)
+void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
+ size_t size)
{
- if (pool->cheap_dma)
- return;
-
dma_sync_single_range_for_device(pool->dev, dma, 0,
size, DMA_BIDIRECTIONAL);
}
-EXPORT_SYMBOL(xp_dma_sync_for_device);
+EXPORT_SYMBOL(xp_dma_sync_for_device_slow);
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index 16bf15864788..5b5d24d2dd37 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -118,6 +118,51 @@ static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr)
return false;
}
+static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ u64 chunk, chunk_end;
+
+ chunk = xp_aligned_extract_addr(pool, desc->addr);
+ chunk_end = xp_aligned_extract_addr(pool, desc->addr + desc->len);
+ if (chunk != chunk_end)
+ return false;
+
+ if (chunk >= pool->addrs_cnt)
+ return false;
+
+ if (desc->options)
+ return false;
+ return true;
+}
+
+static inline bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ u64 addr, base_addr;
+
+ base_addr = xp_unaligned_extract_addr(desc->addr);
+ addr = xp_unaligned_add_offset_to_addr(desc->addr);
+
+ if (desc->len > pool->chunk_size)
+ return false;
+
+ if (base_addr >= pool->addrs_cnt || addr >= pool->addrs_cnt ||
+ xp_desc_crosses_non_contig_pg(pool, addr, desc->len))
+ return false;
+
+ if (desc->options)
+ return false;
+ return true;
+}
+
+static inline bool xp_validate_desc(struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ return pool->unaligned ? xp_unaligned_validate_desc(pool, desc) :
+ xp_aligned_validate_desc(pool, desc);
+}
+
static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
struct xdp_desc *d,
struct xdp_umem *umem)
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread* [PATCH bpf-next 14/14] MAINTAINERS, xsk: update AF_XDP section after moves/adds
2020-05-07 10:42 [PATCH bpf-next 00/14] Introduce AF_XDP buffer allocation API Björn Töpel
` (12 preceding siblings ...)
2020-05-07 10:42 ` [PATCH bpf-next 13/14] xsk: explicitly inline functions and move definitions Björn Töpel
@ 2020-05-07 10:42 ` Björn Töpel
13 siblings, 0 replies; 24+ messages in thread
From: Björn Töpel @ 2020-05-07 10:42 UTC (permalink / raw)
To: ast, daniel, davem, kuba, hawk, john.fastabend, netdev, bpf,
magnus.karlsson, jonathan.lemon, jeffrey.t.kirsher
Cc: Björn Töpel, maximmi, maciej.fijalkowski, Joe Perches
From: Björn Töpel <bjorn.topel@intel.com>
Update MAINTAINERS to correctly mirror the current AF_XDP socket file
layout. Also, add the AF_XDP files of libbpf.
rfc->v1: Sorted file entries. (Joe)
Cc: Joe Perches <joe@perches.com>
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
---
MAINTAINERS | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/MAINTAINERS b/MAINTAINERS
index db7a6d462dff..79e2bb1280e6 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -18451,8 +18451,12 @@ R: Jonathan Lemon <jonathan.lemon@gmail.com>
L: netdev@vger.kernel.org
L: bpf@vger.kernel.org
S: Maintained
-F: kernel/bpf/xskmap.c
+F: include/net/xdp_sock*
+F: include/net/xsk_buffer_pool.h
+F: include/uapi/linux/if_xdp.h
F: net/xdp/
+F: samples/bpf/xdpsock*
+F: tools/lib/bpf/xsk*
XEN BLOCK SUBSYSTEM
M: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
--
2.25.1
^ permalink raw reply related [flat|nested] 24+ messages in thread