All of lore.kernel.org
 help / color / mirror / Atom feed
From: Paolo Abeni <pabeni@redhat.com>
To: netdev@vger.kernel.org
Cc: "David S. Miller" <davem@davemloft.net>,
	Eric Dumazet <eric.dumazet@gmail.com>
Subject: [PATCH net-next 3/4] net: use indirect call wrapper at GRO transport layer
Date: Mon,  3 Dec 2018 12:40:23 +0100	[thread overview]
Message-ID: <01441becf046011b9fc61254aac2421e77f3d5fd.1543836967.git.pabeni@redhat.com> (raw)
In-Reply-To: <cover.1543836966.git.pabeni@redhat.com>

This avoids an indirect call in the receive path for TCP and UDP
packets. TCP takes precedence on UDP, so that we have a single
additional conditional in the common case.

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
---
 include/net/inet_common.h |  7 +++++++
 net/ipv4/af_inet.c        | 11 +++++++++--
 net/ipv4/tcp_offload.c    |  5 +++++
 net/ipv4/udp_offload.c    |  5 +++++
 net/ipv6/ip6_offload.c    | 10 ++++++++--
 net/ipv6/tcpv6_offload.c  |  5 +++++
 net/ipv6/udp_offload.c    |  5 +++++
 7 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/include/net/inet_common.h b/include/net/inet_common.h
index 56e7592811ea..667bb8247f9a 100644
--- a/include/net/inet_common.h
+++ b/include/net/inet_common.h
@@ -56,4 +56,11 @@ static inline void inet_ctl_sock_destroy(struct sock *sk)
 		sock_release(sk->sk_socket);
 }
 
+#define indirect_call_gro_receive(name, cb, head, skb)	\
+({							\
+	unlikely(gro_recursion_inc_test(skb)) ?		\
+		NAPI_GRO_CB(skb)->flush |= 1, NULL :	\
+		INDIRECT_CALL_2(cb, name, head, skb);	\
+})
+
 #endif
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 04ab7ebd6e9b..774f183f56e3 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1385,6 +1385,8 @@ struct sk_buff *inet_gso_segment(struct sk_buff *skb,
 }
 EXPORT_SYMBOL(inet_gso_segment);
 
+INDIRECT_CALLABLE_DECLARE_2(struct sk_buff *, transport4_gro_receive,
+			    struct list_head *head, struct sk_buff *skb);
 struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 {
 	const struct net_offload *ops;
@@ -1494,7 +1496,8 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 	skb_gro_pull(skb, sizeof(*iph));
 	skb_set_transport_header(skb, skb_gro_offset(skb));
 
-	pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
+	pp = indirect_call_gro_receive(transport4_gro_receive,
+				       ops->callbacks.gro_receive, head, skb);
 
 out_unlock:
 	rcu_read_unlock();
@@ -1558,6 +1561,8 @@ int inet_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len)
 	return -EINVAL;
 }
 
+INDIRECT_CALLABLE_DECLARE_2(int, transport4_gro_complete, struct sk_buff *skb,
+			    int);
 int inet_gro_complete(struct sk_buff *skb, int nhoff)
 {
 	__be16 newlen = htons(skb->len - nhoff);
@@ -1583,7 +1588,9 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff)
 	 * because any hdr with option will have been flushed in
 	 * inet_gro_receive().
 	 */
-	err = ops->callbacks.gro_complete(skb, nhoff + sizeof(*iph));
+	err = INDIRECT_CALL_2(ops->callbacks.gro_complete,
+			      transport4_gro_complete, skb,
+			      nhoff + sizeof(*iph));
 
 out_unlock:
 	rcu_read_unlock();
diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c
index 870b0a335061..3d5dfac4cd1b 100644
--- a/net/ipv4/tcp_offload.c
+++ b/net/ipv4/tcp_offload.c
@@ -10,6 +10,7 @@
  *	TCPv4 GSO/GRO support
  */
 
+#include <linux/indirect_call_wrapper.h>
 #include <linux/skbuff.h>
 #include <net/tcp.h>
 #include <net/protocol.h>
@@ -317,6 +318,8 @@ static struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *
 
 	return tcp_gro_receive(head, skb);
 }
+INDIRECT_CALLABLE(tcp4_gro_receive, 2, struct sk_buff *, transport4_gro_receive,
+		  struct list_head *, struct sk_buff *);
 
 static int tcp4_gro_complete(struct sk_buff *skb, int thoff)
 {
@@ -332,6 +335,8 @@ static int tcp4_gro_complete(struct sk_buff *skb, int thoff)
 
 	return tcp_gro_complete(skb);
 }
+INDIRECT_CALLABLE(tcp4_gro_complete, 2, int, transport4_gro_complete,
+		  struct sk_buff *, int);
 
 static const struct net_offload tcpv4_offload = {
 	.callbacks = {
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index 0646d61f4fa8..c3c5b237c8e0 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -13,6 +13,7 @@
 #include <linux/skbuff.h>
 #include <net/udp.h>
 #include <net/protocol.h>
+#include <net/inet_common.h>
 
 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
 	netdev_features_t features,
@@ -477,6 +478,8 @@ static struct sk_buff *udp4_gro_receive(struct list_head *head,
 	NAPI_GRO_CB(skb)->flush = 1;
 	return NULL;
 }
+INDIRECT_CALLABLE(udp4_gro_receive, 1, struct sk_buff *, transport4_gro_receive,
+		  struct list_head *, struct sk_buff *);
 
 static int udp_gro_complete_segment(struct sk_buff *skb)
 {
@@ -536,6 +539,8 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
 
 	return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
 }
+INDIRECT_CALLABLE(udp4_gro_complete, 1, int, transport4_gro_complete,
+		  struct sk_buff *, int);
 
 static const struct net_offload udpv4_offload = {
 	.callbacks = {
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index a1c2bfb2ce0d..eeca4164a155 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -164,6 +164,8 @@ static int ipv6_exthdrs_len(struct ipv6hdr *iph,
 	return len;
 }
 
+INDIRECT_CALLABLE_DECLARE_2(struct sk_buff *, transport6_gro_receive,
+			    struct list_head *head, struct sk_buff *skb);
 static struct sk_buff *ipv6_gro_receive(struct list_head *head,
 					struct sk_buff *skb)
 {
@@ -260,7 +262,8 @@ static struct sk_buff *ipv6_gro_receive(struct list_head *head,
 
 	skb_gro_postpull_rcsum(skb, iph, nlen);
 
-	pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
+	pp = indirect_call_gro_receive(transport6_gro_receive,
+				       ops->callbacks.gro_receive, head, skb);
 
 out_unlock:
 	rcu_read_unlock();
@@ -303,6 +306,8 @@ static struct sk_buff *ip4ip6_gro_receive(struct list_head *head,
 	return inet_gro_receive(head, skb);
 }
 
+INDIRECT_CALLABLE_DECLARE_2(int, transport6_gro_complete, struct sk_buff *skb,
+			    int);
 static int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
 {
 	const struct net_offload *ops;
@@ -322,7 +327,8 @@ static int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
 	if (WARN_ON(!ops || !ops->callbacks.gro_complete))
 		goto out_unlock;
 
-	err = ops->callbacks.gro_complete(skb, nhoff);
+	err = INDIRECT_CALL_2(ops->callbacks.gro_complete,
+			      transport6_gro_complete, skb, nhoff);
 
 out_unlock:
 	rcu_read_unlock();
diff --git a/net/ipv6/tcpv6_offload.c b/net/ipv6/tcpv6_offload.c
index e72947c99454..3c85afc4cf43 100644
--- a/net/ipv6/tcpv6_offload.c
+++ b/net/ipv6/tcpv6_offload.c
@@ -9,6 +9,7 @@
  *
  *      TCPv6 GSO/GRO support
  */
+#include <linux/indirect_call_wrapper.h>
 #include <linux/skbuff.h>
 #include <net/protocol.h>
 #include <net/tcp.h>
@@ -28,6 +29,8 @@ static struct sk_buff *tcp6_gro_receive(struct list_head *head,
 
 	return tcp_gro_receive(head, skb);
 }
+INDIRECT_CALLABLE(tcp6_gro_receive, 2, struct sk_buff *, transport6_gro_receive,
+		  struct list_head *, struct sk_buff *);
 
 static int tcp6_gro_complete(struct sk_buff *skb, int thoff)
 {
@@ -40,6 +43,8 @@ static int tcp6_gro_complete(struct sk_buff *skb, int thoff)
 
 	return tcp_gro_complete(skb);
 }
+INDIRECT_CALLABLE(tcp6_gro_complete, 2, int, transport6_gro_complete,
+		  struct sk_buff *, int);
 
 static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
 					netdev_features_t features)
diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c
index 828b2457f97b..ce4d491c583c 100644
--- a/net/ipv6/udp_offload.c
+++ b/net/ipv6/udp_offload.c
@@ -11,6 +11,7 @@
  */
 #include <linux/skbuff.h>
 #include <linux/netdevice.h>
+#include <linux/indirect_call_wrapper.h>
 #include <net/protocol.h>
 #include <net/ipv6.h>
 #include <net/udp.h>
@@ -141,6 +142,8 @@ static struct sk_buff *udp6_gro_receive(struct list_head *head,
 	NAPI_GRO_CB(skb)->flush = 1;
 	return NULL;
 }
+INDIRECT_CALLABLE(udp6_gro_receive, 1, struct sk_buff *, transport6_gro_receive,
+		  struct list_head *, struct sk_buff *);
 
 static int udp6_gro_complete(struct sk_buff *skb, int nhoff)
 {
@@ -153,6 +156,8 @@ static int udp6_gro_complete(struct sk_buff *skb, int nhoff)
 
 	return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
 }
+INDIRECT_CALLABLE(udp6_gro_complete, 1, int, transport6_gro_complete,
+		  struct sk_buff *, int);
 
 static const struct net_offload udpv6_offload = {
 	.callbacks = {
-- 
2.19.2

  parent reply	other threads:[~2018-12-03 11:41 UTC|newest]

Thread overview: 13+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-12-03 11:40 [PATCH net-next 0/4] net: mitigate retpoline overhead Paolo Abeni
2018-12-03 11:40 ` [PATCH net-next 1/4] indirect call wrappers: helpers to speed-up indirect calls of builtin Paolo Abeni
2018-12-03 18:04   ` Eric Dumazet
2018-12-04 11:27     ` Paolo Abeni
2018-12-04 16:51       ` David Miller
2018-12-04 17:13   ` Edward Cree
2018-12-04 17:44     ` Paolo Abeni
2018-12-04 18:12       ` Edward Cree
2018-12-05 18:03         ` Paolo Abeni
2018-12-03 11:40 ` [PATCH net-next 2/4] net: use indirect call wrappers at GRO network layer Paolo Abeni
2018-12-03 11:40 ` Paolo Abeni [this message]
2018-12-07  3:48   ` [PATCH net-next 3/4] net: use indirect call wrapper at GRO transport layer kbuild test robot
2018-12-03 11:40 ` [PATCH net-next 4/4] udp: use indirect call wrapper for GRO socket lookup Paolo Abeni

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=01441becf046011b9fc61254aac2421e77f3d5fd.1543836967.git.pabeni@redhat.com \
    --to=pabeni@redhat.com \
    --cc=davem@davemloft.net \
    --cc=eric.dumazet@gmail.com \
    --cc=netdev@vger.kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.