From: Prashant Bhole <prashantbhole.linux@gmail.com>
To: "David S . Miller" <davem@davemloft.net>,
"Michael S . Tsirkin" <mst@redhat.com>
Cc: Prashant Bhole <prashantbhole.linux@gmail.com>,
Jason Wang <jasowang@redhat.com>,
Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Jakub Kicinski <jakub.kicinski@netronome.com>,
Jesper Dangaard Brouer <hawk@kernel.org>,
John Fastabend <john.fastabend@gmail.com>,
Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>,
Yonghong Song <yhs@fb.com>, Andrii Nakryiko <andriin@fb.com>,
netdev@vger.kernel.org, qemu-devel@nongnu.org,
kvm@vger.kernel.org
Subject: [RFC net-next 08/18] tun: run offloaded XDP program in Tx path
Date: Tue, 26 Nov 2019 19:07:34 +0900 [thread overview]
Message-ID: <20191126100744.5083-9-prashantbhole.linux@gmail.com> (raw)
In-Reply-To: <20191126100744.5083-1-prashantbhole.linux@gmail.com>
run offloaded XDP program as soon as packet is removed from the ptr
ring. Since this is XDP in Tx path, the traditional handling of
XDP actions XDP_TX/REDIRECT isn't valid. For this reason we call
do_xdp_generic_core instead of do_xdp_generic. do_xdp_generic_core
just runs the program and leaves the action handling to us.
Signed-off-by: Prashant Bhole <prashantbhole.linux@gmail.com>
---
drivers/net/tun.c | 149 +++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 146 insertions(+), 3 deletions(-)
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index ecb49101b0b5..466ea69f00ee 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -131,6 +131,7 @@ struct tap_filter {
/* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
* to max number of VCPUs in guest. */
#define MAX_TAP_QUEUES 256
+#define MAX_TAP_BATCH 64
#define MAX_TAP_FLOWS 4096
#define TUN_FLOW_EXPIRE (3 * HZ)
@@ -2156,6 +2157,109 @@ static ssize_t tun_put_user(struct tun_struct *tun,
return total;
}
+static struct sk_buff *tun_prepare_xdp_skb(struct sk_buff *skb)
+{
+ struct sk_buff *nskb;
+
+ if (skb_shared(skb) || skb_cloned(skb)) {
+ nskb = skb_copy(skb, GFP_ATOMIC);
+ consume_skb(skb);
+ return nskb;
+ }
+
+ return skb;
+}
+
+static u32 tun_do_xdp_offload_generic(struct tun_struct *tun,
+ struct sk_buff *skb)
+{
+ struct tun_prog *xdp_prog;
+ struct xdp_buff xdp;
+ u32 act = XDP_PASS;
+
+ xdp_prog = rcu_dereference(tun->offloaded_xdp_prog);
+ if (xdp_prog) {
+ skb = tun_prepare_xdp_skb(skb);
+ if (!skb) {
+ act = XDP_DROP;
+ kfree_skb(skb);
+ goto drop;
+ }
+
+ act = do_xdp_generic_core(skb, &xdp, xdp_prog->prog);
+ switch (act) {
+ case XDP_TX:
+ /*
+ * Rx path generic XDP will be called in this path
+ */
+ netif_receive_skb(skb);
+ break;
+ case XDP_PASS:
+ break;
+ case XDP_REDIRECT:
+ /*
+ * Since we are not handling this case yet, let's free
+ * skb here. In case of XDP_DROP/XDP_ABORTED, the skb
+ * was already freed in do_xdp_generic_core()
+ */
+ kfree_skb(skb);
+ /* fall through */
+ default:
+ bpf_warn_invalid_xdp_action(act);
+ /* fall through */
+ case XDP_ABORTED:
+ trace_xdp_exception(tun->dev, xdp_prog->prog, act);
+ /* fall through */
+ case XDP_DROP:
+ goto drop;
+ }
+ }
+
+ return act;
+drop:
+ this_cpu_inc(tun->pcpu_stats->tx_dropped);
+ return act;
+}
+
+static u32 tun_do_xdp_offload(struct tun_struct *tun, struct tun_file *tfile,
+ struct xdp_frame *frame)
+{
+ struct tun_prog *xdp_prog;
+ struct tun_page tpage;
+ struct xdp_buff xdp;
+ u32 act = XDP_PASS;
+ int flush = 0;
+
+ xdp_prog = rcu_dereference(tun->offloaded_xdp_prog);
+ if (xdp_prog) {
+ xdp.data_hard_start = frame->data - frame->headroom;
+ xdp.data = frame->data;
+ xdp.data_end = xdp.data + frame->len;
+ xdp.data_meta = xdp.data - frame->metasize;
+
+ act = bpf_prog_run_xdp(xdp_prog->prog, &xdp);
+ switch (act) {
+ case XDP_PASS:
+ break;
+ case XDP_TX:
+ /* fall through */
+ case XDP_REDIRECT:
+ /* fall through */
+ default:
+ bpf_warn_invalid_xdp_action(act);
+ /* fall through */
+ case XDP_ABORTED:
+ trace_xdp_exception(tun->dev, xdp_prog->prog, act);
+ /* fall through */
+ case XDP_DROP:
+ xdp_return_frame_rx_napi(frame);
+ break;
+ }
+ }
+
+ return act;
+}
+
static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
{
DECLARE_WAITQUEUE(wait, current);
@@ -2574,6 +2678,47 @@ static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
return ret;
}
+static int tun_consume_packets(struct tun_file *tfile, void **ptr_array, int n)
+{
+ struct tun_prog *xdp_prog;
+ struct xdp_frame *frame;
+ struct tun_struct *tun;
+ int i, num_ptrs;
+ int pkt_cnt = 0;
+ void *pkts[MAX_TAP_BATCH];
+ void *ptr;
+ u32 act;
+
+ if (unlikely(!tfile))
+ return 0;
+
+ if (n > MAX_TAP_BATCH)
+ n = MAX_TAP_BATCH;
+
+ rcu_read_lock();
+ tun = rcu_dereference(tfile->tun);
+ if (unlikely(!tun))
+ return 0;
+ xdp_prog = rcu_dereference(tun->offloaded_xdp_prog);
+
+ num_ptrs = ptr_ring_consume_batched(&tfile->tx_ring, pkts, n);
+ for (i = 0; i < num_ptrs; i++) {
+ ptr = pkts[i];
+ if (tun_is_xdp_frame(ptr)) {
+ frame = tun_ptr_to_xdp(ptr);
+ act = tun_do_xdp_offload(tun, tfile, frame);
+ } else {
+ act = tun_do_xdp_offload_generic(tun, ptr);
+ }
+
+ if (act == XDP_PASS)
+ ptr_array[pkt_cnt++] = ptr;
+ }
+
+ rcu_read_unlock();
+ return pkt_cnt;
+}
+
static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
int flags)
{
@@ -2594,9 +2739,7 @@ static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
ptr = ctl->ptr;
break;
case TUN_MSG_CONSUME_PKTS:
- ret = ptr_ring_consume_batched(&tfile->tx_ring,
- ctl->ptr,
- ctl->num);
+ ret = tun_consume_packets(tfile, ctl->ptr, ctl->num);
goto out;
case TUN_MSG_UNCONSUME_PKTS:
ptr_ring_unconsume(&tfile->tx_ring, ctl->ptr,
--
2.20.1
next prev parent reply other threads:[~2019-11-26 10:09 UTC|newest]
Thread overview: 43+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-11-26 10:07 [RFC net-next 00/18] virtio_net XDP offload Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 01/18] bpf: introduce bpf_prog_offload_verifier_setup() Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 02/18] net: core: rename netif_receive_generic_xdp() to do_generic_xdp_core() Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 03/18] net: core: export do_xdp_generic_core() Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 04/18] tuntap: check tun_msg_ctl type at necessary places Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 05/18] vhost_net: user tap recvmsg api to access ptr ring Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 06/18] tuntap: remove usage of ptr ring in vhost_net Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 07/18] tun: set offloaded xdp program Prashant Bhole
2019-12-01 16:35 ` David Ahern
2019-12-02 2:44 ` Jason Wang
2019-12-01 16:45 ` David Ahern
2019-12-02 2:47 ` Jason Wang
2019-12-09 0:24 ` Prashant Bhole
2019-11-26 10:07 ` Prashant Bhole [this message]
2019-12-01 16:39 ` [RFC net-next 08/18] tun: run offloaded XDP program in Tx path David Ahern
2019-12-01 20:56 ` David Miller
2019-12-01 21:40 ` Michael S. Tsirkin
2019-12-01 21:54 ` David Miller
2019-12-02 2:56 ` Jason Wang
2019-12-02 2:45 ` Jason Wang
2019-11-26 10:07 ` [RFC net-next 09/18] tun: add a way to inject Tx path packet into Rx path Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 10/18] tun: handle XDP_TX action of offloaded program Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 11/18] tun: run xdp prog when tun is read from file interface Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 12/18] virtio-net: store xdp_prog in device Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 13/18] virtio_net: use XDP attachment helpers Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 14/18] virtio_net: add XDP prog offload infrastructure Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 15/18] virtio_net: implement XDP prog offload functionality Prashant Bhole
2019-11-27 20:42 ` Michael S. Tsirkin
2019-11-28 2:53 ` Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 16/18] bpf: export function __bpf_map_get Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 17/18] virtio_net: implment XDP map offload functionality Prashant Bhole
2019-11-26 10:07 ` [RFC net-next 18/18] virtio_net: restrict bpf helper calls from offloaded program Prashant Bhole
2019-11-26 20:35 ` [RFC net-next 00/18] virtio_net XDP offload Jakub Kicinski
2019-11-27 2:59 ` Jason Wang
2019-11-27 19:49 ` Jakub Kicinski
2019-11-28 3:41 ` Jason Wang
2019-11-27 20:32 ` Michael S. Tsirkin
2019-11-27 23:40 ` Jakub Kicinski
2019-12-02 15:29 ` Michael S. Tsirkin
2019-11-28 3:32 ` Alexei Starovoitov
2019-11-28 4:18 ` Jason Wang
2019-12-01 16:54 ` David Ahern
2019-12-02 2:48 ` Jason Wang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191126100744.5083-9-prashantbhole.linux@gmail.com \
--to=prashantbhole.linux@gmail.com \
--cc=andriin@fb.com \
--cc=ast@kernel.org \
--cc=daniel@iogearbox.net \
--cc=davem@davemloft.net \
--cc=hawk@kernel.org \
--cc=jakub.kicinski@netronome.com \
--cc=jasowang@redhat.com \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kvm@vger.kernel.org \
--cc=mst@redhat.com \
--cc=netdev@vger.kernel.org \
--cc=qemu-devel@nongnu.org \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).