[PATCH net-next 0/4] tcp: better smp listener behavior

[PATCH net-next 0/4] tcp: better smp listener behavior

[Eric Dumazet]

As promised in last patch series, we implement a better SO_REUSEPORT
strategy, based on cpu affinities if selected by the application.

We also moved sk_refcnt out of the cache line containing the lookup
keys, as it was considerably slowing down smp operations because
of false sharing. This was simpler than converting listen sockets
to conventional RCU (to avoid sk_refcnt dirtying)

Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.

Eric Dumazet (4):
  net: SO_INCOMING_CPU setsockopt() support
  net: align sk_refcnt on 128 bytes boundary
  net: shrink struct sock and request_sock by 8 bytes
  tcp: shrink tcp_timewait_sock by 8 bytes

 include/linux/tcp.h              |  4 ++--
 include/net/inet_timewait_sock.h |  2 +-
 include/net/request_sock.h       |  7 +++----
 include/net/sock.h               | 37 +++++++++++++++++++++++++++----------
 net/core/sock.c                  |  5 +++++
 net/ipv4/inet_hashtables.c       |  5 +++++
 net/ipv4/syncookies.c            |  4 ++--
 net/ipv4/tcp_input.c             |  2 +-
 net/ipv4/tcp_ipv4.c              |  2 +-
 net/ipv4/tcp_minisocks.c         | 18 +++++++++---------
 net/ipv4/tcp_output.c            |  2 +-
 net/ipv4/udp.c                   | 12 +++++++++++-
 net/ipv6/inet6_hashtables.c      |  5 +++++
 net/ipv6/syncookies.c            |  4 ++--
 net/ipv6/tcp_ipv6.c              |  2 +-
 net/ipv6/udp.c                   | 11 +++++++++++
 16 files changed, 87 insertions(+), 35 deletions(-)

-- 
2.6.0.rc2.230.g3dd15c0

[PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Eric Dumazet]

SO_INCOMING_CPU as added in commit 2c8c56e15df3 was a getsockopt() command
to fetch incoming cpu handling a particular TCP flow after accept()

This commits adds setsockopt() support and extends SO_REUSEPORT selection
logic : If a TCP listener or UDP socket has this option set, a packet is
delivered to this socket only if CPU handling the packet matches the specified one.

This allows to build very efficient TCP servers, using one thread per cpu,
as the associated TCP listener should only accept flows handled in softirq
by the same cpu. This provides optimal NUMA/SMP behavior and keep cpu caches hot.

Note that __inet_lookup_listener() still has to iterate over the list of
all listeners. Following patch puts sk_refcnt in a different cache line
to let this iteration hit only shared and read mostly cache lines.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/sock.h          | 11 +++++------
 net/core/sock.c             |  5 +++++
 net/ipv4/inet_hashtables.c  |  5 +++++
 net/ipv4/udp.c              | 12 +++++++++++-
 net/ipv6/inet6_hashtables.c |  5 +++++
 net/ipv6/udp.c              | 11 +++++++++++
 6 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/include/net/sock.h b/include/net/sock.h
index dfe2eb8e1132..00f60bea983b 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -150,6 +150,7 @@ typedef __u64 __bitwise __addrpair;
  *	@skc_node: main hash linkage for various protocol lookup tables
  *	@skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
  *	@skc_tx_queue_mapping: tx queue number for this connection
+ *	@skc_incoming_cpu: record/match cpu processing incoming packets
  *	@skc_refcnt: reference count
  *
  *	This is the minimal network layer representation of sockets, the header
@@ -212,6 +213,8 @@ struct sock_common {
 		struct hlist_nulls_node skc_nulls_node;
 	};
 	int			skc_tx_queue_mapping;
+	int			skc_incoming_cpu;
+
 	atomic_t		skc_refcnt;
 	/* private: */
 	int                     skc_dontcopy_end[0];
@@ -274,7 +277,7 @@ struct cg_proto;
   *	@sk_rcvtimeo: %SO_RCVTIMEO setting
   *	@sk_sndtimeo: %SO_SNDTIMEO setting
   *	@sk_rxhash: flow hash received from netif layer
-  *	@sk_incoming_cpu: record cpu processing incoming packets
+  *	@sk_incoming_cpu: record/match cpu processing incoming packets
   *	@sk_txhash: computed flow hash for use on transmit
   *	@sk_filter: socket filtering instructions
   *	@sk_timer: sock cleanup timer
@@ -331,6 +334,7 @@ struct sock {
 #define sk_v6_daddr		__sk_common.skc_v6_daddr
 #define sk_v6_rcv_saddr	__sk_common.skc_v6_rcv_saddr
 #define sk_cookie		__sk_common.skc_cookie
+#define sk_incoming_cpu		__sk_common.skc_incoming_cpu
 
 	socket_lock_t		sk_lock;
 	struct sk_buff_head	sk_receive_queue;
@@ -353,11 +357,6 @@ struct sock {
 #ifdef CONFIG_RPS
 	__u32			sk_rxhash;
 #endif
-	u16			sk_incoming_cpu;
-	/* 16bit hole
-	 * Warned : sk_incoming_cpu can be set from softirq,
-	 * Do not use this hole without fully understanding possible issues.
-	 */
 
 	__u32			sk_txhash;
 #ifdef CONFIG_NET_RX_BUSY_POLL
diff --git a/net/core/sock.c b/net/core/sock.c
index 7dd1263e4c24..1071f9380250 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -988,6 +988,10 @@ set_rcvbuf:
 					 sk->sk_max_pacing_rate);
 		break;
 
+	case SO_INCOMING_CPU:
+		sk->sk_incoming_cpu = val;
+		break;
+
 	default:
 		ret = -ENOPROTOOPT;
 		break;
@@ -2353,6 +2357,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
 
 	sk->sk_max_pacing_rate = ~0U;
 	sk->sk_pacing_rate = ~0U;
+	sk->sk_incoming_cpu = -1;
 	/*
 	 * Before updating sk_refcnt, we must commit prior changes to memory
 	 * (Documentation/RCU/rculist_nulls.txt for details)
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index bed8886a4b6c..eabcfbc13afb 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -185,6 +185,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
 				return -1;
 			score += 4;
 		}
+		if (sk->sk_incoming_cpu != -1) {
+			if (sk->sk_incoming_cpu != raw_smp_processor_id())
+				return -1;
+			score++;
+		}
 	}
 	return score;
 }
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index e1fc129099ea..de675b796f78 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -375,7 +375,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
 			return -1;
 		score += 4;
 	}
-
+	if (sk->sk_incoming_cpu != -1) {
+		if (sk->sk_incoming_cpu != raw_smp_processor_id())
+			return -1;
+		score++;
+	}
 	return score;
 }
 
@@ -419,6 +423,12 @@ static inline int compute_score2(struct sock *sk, struct net *net,
 		score += 4;
 	}
 
+	if (sk->sk_incoming_cpu != -1) {
+		if (sk->sk_incoming_cpu != raw_smp_processor_id())
+			return -1;
+		score++;
+	}
+
 	return score;
 }
 
diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 6ac8dad0138a..af3d7f826bff 100644
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -114,6 +114,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
 				return -1;
 			score++;
 		}
+		if (sk->sk_incoming_cpu != -1) {
+			if (sk->sk_incoming_cpu != raw_smp_processor_id())
+				return -1;
+			score++;
+		}
 	}
 	return score;
 }
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 0aba654f5b91..222fdc780405 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -182,6 +182,12 @@ static inline int compute_score(struct sock *sk, struct net *net,
 		score++;
 	}
 
+	if (sk->sk_incoming_cpu != -1) {
+		if (sk->sk_incoming_cpu != raw_smp_processor_id())
+			return -1;
+		score++;
+	}
+
 	return score;
 }
 
@@ -223,6 +229,11 @@ static inline int compute_score2(struct sock *sk, struct net *net,
 		score++;
 	}
 
+	if (sk->sk_incoming_cpu != -1) {
+		if (sk->sk_incoming_cpu != raw_smp_processor_id())
+			return -1;
+		score++;
+	}
 	return score;
 }
 
-- 
2.6.0.rc2.230.g3dd15c0

[PATCH net-next 2/4] net: align sk_refcnt on 128 bytes boundary

[Eric Dumazet]

sk->sk_refcnt is dirtied for every TCP/UDP incoming packet.
This is a performance issue if multiple cpus hit a common socket,
or multiple sockets are chained due to SO_REUSEPORT.

By moving sk_recnt 8 bytes further, first 128 bytes of sockets
are mostly read. As they contain the lookup keys, this has
a considerable performance impact, as cpus can cache them.

These 8 bytes are not wasted, we use them as a place holder
for various fields, depending on the socket type.

Tested:
 SYN flood hitting a 16 RX queues NIC.
 TCP listener using 16 sockets and SO_REUSEPORT
 and SO_INCOMING_CPU for proper siloing.

 Could process 6.0 Mpps SYN instead of 4.2 Mpps

 Kernel profile looked like :
    11.68%  [kernel]  [k] sha_transform
     6.51%  [kernel]  [k] __inet_lookup_listener
     5.07%  [kernel]  [k] __inet_lookup_established
     4.15%  [kernel]  [k] memcpy_erms
     3.46%  [kernel]  [k] ipt_do_table
     2.74%  [kernel]  [k] fib_table_lookup
     2.54%  [kernel]  [k] tcp_make_synack
     2.34%  [kernel]  [k] tcp_conn_request
     2.05%  [kernel]  [k] __netif_receive_skb_core
     2.03%  [kernel]  [k] kmem_cache_alloc

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/inet_timewait_sock.h |  2 +-
 include/net/request_sock.h       |  2 +-
 include/net/sock.h               | 13 ++++++++++++-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h
index 186f3a1e1b1f..e581fc69129d 100644
--- a/include/net/inet_timewait_sock.h
+++ b/include/net/inet_timewait_sock.h
@@ -70,6 +70,7 @@ struct inet_timewait_sock {
 #define tw_dport		__tw_common.skc_dport
 #define tw_num			__tw_common.skc_num
 #define tw_cookie		__tw_common.skc_cookie
+#define tw_dr			__tw_common.skc_tw_dr
 
 	int			tw_timeout;
 	volatile unsigned char	tw_substate;
@@ -88,7 +89,6 @@ struct inet_timewait_sock {
 	kmemcheck_bitfield_end(flags);
 	struct timer_list	tw_timer;
 	struct inet_bind_bucket	*tw_tb;
-	struct inet_timewait_death_row *tw_dr;
 };
 #define tw_tclass tw_tos
 
diff --git a/include/net/request_sock.h b/include/net/request_sock.h
index 95ab5d7aab96..6b818b77d5e5 100644
--- a/include/net/request_sock.h
+++ b/include/net/request_sock.h
@@ -50,9 +50,9 @@ struct request_sock {
 	struct sock_common		__req_common;
 #define rsk_refcnt			__req_common.skc_refcnt
 #define rsk_hash			__req_common.skc_hash
+#define rsk_listener			__req_common.skc_listener
 
 	struct request_sock		*dl_next;
-	struct sock			*rsk_listener;
 	u16				mss;
 	u8				num_retrans; /* number of retransmits */
 	u8				cookie_ts:1; /* syncookie: encode tcpopts in timestamp */
diff --git a/include/net/sock.h b/include/net/sock.h
index 00f60bea983b..2ab099280979 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -150,6 +150,7 @@ typedef __u64 __bitwise __addrpair;
  *	@skc_node: main hash linkage for various protocol lookup tables
  *	@skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
  *	@skc_tx_queue_mapping: tx queue number for this connection
+ *	@skc_flags: place holder for sk_flags
  *	@skc_incoming_cpu: record/match cpu processing incoming packets
  *	@skc_refcnt: reference count
  *
@@ -201,6 +202,16 @@ struct sock_common {
 
 	atomic64_t		skc_cookie;
 
+	/* following fields are padding to force
+	 * offset(struct sock, sk_refcnt) == 128 on 64bit arches
+	 * assuming IPV6 is enabled. We use this padding differently
+	 * for different kind of 'sockets'
+	 */
+	union {
+		unsigned long	skc_flags;
+		struct sock	*skc_listener; /* request_sock */
+		struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */
+	};
 	/*
 	 * fields between dontcopy_begin/dontcopy_end
 	 * are not copied in sock_copy()
@@ -335,6 +346,7 @@ struct sock {
 #define sk_v6_rcv_saddr	__sk_common.skc_v6_rcv_saddr
 #define sk_cookie		__sk_common.skc_cookie
 #define sk_incoming_cpu		__sk_common.skc_incoming_cpu
+#define sk_flags		__sk_common.skc_flags
 
 	socket_lock_t		sk_lock;
 	struct sk_buff_head	sk_receive_queue;
@@ -372,7 +384,6 @@ struct sock {
 #ifdef CONFIG_XFRM
 	struct xfrm_policy	*sk_policy[2];
 #endif
-	unsigned long 		sk_flags;
 	struct dst_entry	*sk_rx_dst;
 	struct dst_entry __rcu	*sk_dst_cache;
 	spinlock_t		sk_dst_lock;
-- 
2.6.0.rc2.230.g3dd15c0

[PATCH net-next 3/4] net: shrink struct sock and request_sock by 8 bytes

[Eric Dumazet]

One 32bit hole is following skc_refcnt, use it.
skc_incoming_cpu can also be an union for request_sock rcv_wnd.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/request_sock.h |  5 ++---
 include/net/sock.h         | 13 +++++++++----
 net/ipv4/syncookies.c      |  4 ++--
 net/ipv4/tcp_input.c       |  2 +-
 net/ipv4/tcp_ipv4.c        |  2 +-
 net/ipv4/tcp_minisocks.c   | 18 +++++++++---------
 net/ipv4/tcp_output.c      |  2 +-
 net/ipv6/syncookies.c      |  4 ++--
 net/ipv6/tcp_ipv6.c        |  2 +-
 9 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/include/net/request_sock.h b/include/net/request_sock.h
index 6b818b77d5e5..2e73748956d5 100644
--- a/include/net/request_sock.h
+++ b/include/net/request_sock.h
@@ -51,15 +51,14 @@ struct request_sock {
 #define rsk_refcnt			__req_common.skc_refcnt
 #define rsk_hash			__req_common.skc_hash
 #define rsk_listener			__req_common.skc_listener
+#define rsk_window_clamp		__req_common.skc_window_clamp
+#define rsk_rcv_wnd			__req_common.skc_rcv_wnd
 
 	struct request_sock		*dl_next;
 	u16				mss;
 	u8				num_retrans; /* number of retransmits */
 	u8				cookie_ts:1; /* syncookie: encode tcpopts in timestamp */
 	u8				num_timeout:7; /* number of timeouts */
-	/* The following two fields can be easily recomputed I think -AK */
-	u32				window_clamp; /* window clamp at creation time */
-	u32				rcv_wnd;	  /* rcv_wnd offered first time */
 	u32				ts_recent;
 	struct timer_list		rsk_timer;
 	const struct request_sock_ops	*rsk_ops;
diff --git a/include/net/sock.h b/include/net/sock.h
index 2ab099280979..6efba5f998f1 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -224,11 +224,18 @@ struct sock_common {
 		struct hlist_nulls_node skc_nulls_node;
 	};
 	int			skc_tx_queue_mapping;
-	int			skc_incoming_cpu;
+	union {
+		int		skc_incoming_cpu;
+		u32		skc_rcv_wnd;
+	};
 
 	atomic_t		skc_refcnt;
 	/* private: */
 	int                     skc_dontcopy_end[0];
+	union {
+		u32		skc_rxhash;
+		u32		skc_window_clamp;
+	};
 	/* public: */
 };
 
@@ -347,6 +354,7 @@ struct sock {
 #define sk_cookie		__sk_common.skc_cookie
 #define sk_incoming_cpu		__sk_common.skc_incoming_cpu
 #define sk_flags		__sk_common.skc_flags
+#define sk_rxhash		__sk_common.skc_rxhash
 
 	socket_lock_t		sk_lock;
 	struct sk_buff_head	sk_receive_queue;
@@ -366,9 +374,6 @@ struct sock {
 	} sk_backlog;
 #define sk_rmem_alloc sk_backlog.rmem_alloc
 	int			sk_forward_alloc;
-#ifdef CONFIG_RPS
-	__u32			sk_rxhash;
-#endif
 
 	__u32			sk_txhash;
 #ifdef CONFIG_NET_RX_BUSY_POLL
diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c
index 8113c30ccf96..0769248bc0db 100644
--- a/net/ipv4/syncookies.c
+++ b/net/ipv4/syncookies.c
@@ -381,10 +381,10 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb)
 	}
 
 	/* Try to redo what tcp_v4_send_synack did. */
-	req->window_clamp = tp->window_clamp ? :dst_metric(&rt->dst, RTAX_WINDOW);
+	req->rsk_window_clamp = tp->window_clamp ? :dst_metric(&rt->dst, RTAX_WINDOW);
 
 	tcp_select_initial_window(tcp_full_space(sk), req->mss,
-				  &req->rcv_wnd, &req->window_clamp,
+				  &req->rsk_rcv_wnd, &req->rsk_window_clamp,
 				  ireq->wscale_ok, &rcv_wscale,
 				  dst_metric(&rt->dst, RTAX_INITRWND));
 
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index ddadb318e850..3b35c3f4d268 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -6022,7 +6022,7 @@ static void tcp_openreq_init(struct request_sock *req,
 {
 	struct inet_request_sock *ireq = inet_rsk(req);
 
-	req->rcv_wnd = 0;		/* So that tcp_send_synack() knows! */
+	req->rsk_rcv_wnd = 0;		/* So that tcp_send_synack() knows! */
 	req->cookie_ts = 0;
 	tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
 	tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 34310748a365..ddb198392c7f 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -803,7 +803,7 @@ static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
 	 */
 	tcp_v4_send_ack(skb, (sk->sk_state == TCP_LISTEN) ?
 			tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
-			tcp_rsk(req)->rcv_nxt, req->rcv_wnd,
+			tcp_rsk(req)->rcv_nxt, req->rsk_rcv_wnd,
 			tcp_time_stamp,
 			req->ts_recent,
 			0,
diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index 9adf1e2c3170..85272bf50f6e 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -381,18 +381,18 @@ void tcp_openreq_init_rwin(struct request_sock *req,
 
 	window_clamp = READ_ONCE(tp->window_clamp);
 	/* Set this up on the first call only */
-	req->window_clamp = window_clamp ? : dst_metric(dst, RTAX_WINDOW);
+	req->rsk_window_clamp = window_clamp ? : dst_metric(dst, RTAX_WINDOW);
 
 	/* limit the window selection if the user enforce a smaller rx buffer */
 	if (sk_listener->sk_userlocks & SOCK_RCVBUF_LOCK &&
-	    (req->window_clamp > full_space || req->window_clamp == 0))
-		req->window_clamp = full_space;
+	    (req->rsk_window_clamp > full_space || req->rsk_window_clamp == 0))
+		req->rsk_window_clamp = full_space;
 
 	/* tcp_full_space because it is guaranteed to be the first packet */
 	tcp_select_initial_window(full_space,
 		mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
-		&req->rcv_wnd,
-		&req->window_clamp,
+		&req->rsk_rcv_wnd,
+		&req->rsk_window_clamp,
 		ireq->wscale_ok,
 		&rcv_wscale,
 		dst_metric(dst, RTAX_INITRWND));
@@ -512,9 +512,9 @@ struct sock *tcp_create_openreq_child(const struct sock *sk,
 			if (sysctl_tcp_fack)
 				tcp_enable_fack(newtp);
 		}
-		newtp->window_clamp = req->window_clamp;
-		newtp->rcv_ssthresh = req->rcv_wnd;
-		newtp->rcv_wnd = req->rcv_wnd;
+		newtp->window_clamp = req->rsk_window_clamp;
+		newtp->rcv_ssthresh = req->rsk_rcv_wnd;
+		newtp->rcv_wnd = req->rsk_rcv_wnd;
 		newtp->rx_opt.wscale_ok = ireq->wscale_ok;
 		if (newtp->rx_opt.wscale_ok) {
 			newtp->rx_opt.snd_wscale = ireq->snd_wscale;
@@ -707,7 +707,7 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
 	/* RFC793: "first check sequence number". */
 
 	if (paws_reject || !tcp_in_window(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq,
-					  tcp_rsk(req)->rcv_nxt, tcp_rsk(req)->rcv_nxt + req->rcv_wnd)) {
+					  tcp_rsk(req)->rcv_nxt, tcp_rsk(req)->rcv_nxt + req->rsk_rcv_wnd)) {
 		/* Out of window: send ACK and drop. */
 		if (!(flg & TCP_FLAG_RST))
 			req->rsk_ops->send_ack(sk, skb, req);
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 55ed3266b05f..6e79fcb0addb 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -3023,7 +3023,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
 	th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
 
 	/* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
-	th->window = htons(min(req->rcv_wnd, 65535U));
+	th->window = htons(min(req->rsk_rcv_wnd, 65535U));
 	tcp_options_write((__be32 *)(th + 1), NULL, &opts);
 	th->doff = (tcp_header_size >> 2);
 	TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_OUTSEGS);
diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c
index f610b5310b17..bb8f2fa1c7fb 100644
--- a/net/ipv6/syncookies.c
+++ b/net/ipv6/syncookies.c
@@ -235,9 +235,9 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb)
 			goto out_free;
 	}
 
-	req->window_clamp = tp->window_clamp ? :dst_metric(dst, RTAX_WINDOW);
+	req->rsk_window_clamp = tp->window_clamp ? :dst_metric(dst, RTAX_WINDOW);
 	tcp_select_initial_window(tcp_full_space(sk), req->mss,
-				  &req->rcv_wnd, &req->window_clamp,
+				  &req->rsk_rcv_wnd, &req->rsk_window_clamp,
 				  ireq->wscale_ok, &rcv_wscale,
 				  dst_metric(dst, RTAX_INITRWND));
 
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 33334f0c217d..2887c8474b65 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -931,7 +931,7 @@ static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
 	 */
 	tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
 			tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
-			tcp_rsk(req)->rcv_nxt, req->rcv_wnd,
+			tcp_rsk(req)->rcv_nxt, req->rsk_rcv_wnd,
 			tcp_time_stamp, req->ts_recent, sk->sk_bound_dev_if,
 			tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->daddr),
 			0, 0);
-- 
2.6.0.rc2.230.g3dd15c0

[PATCH net-next 4/4] tcp: shrink tcp_timewait_sock by 8 bytes

[Eric Dumazet]

Reducing tcp_timewait_sock from 280 bytes to 272 bytes
allows SLAB to pack 15 objects per page instead of 14 (on x86)

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/linux/tcp.h | 4 ++--
 include/net/sock.h  | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index e442e6e9a365..86a7edaa6797 100644
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -356,8 +356,8 @@ static inline struct tcp_sock *tcp_sk(const struct sock *sk)
 
 struct tcp_timewait_sock {
 	struct inet_timewait_sock tw_sk;
-	u32			  tw_rcv_nxt;
-	u32			  tw_snd_nxt;
+#define tw_rcv_nxt tw_sk.__tw_common.skc_tw_rcv_nxt
+#define tw_snd_nxt tw_sk.__tw_common.skc_tw_snd_nxt
 	u32			  tw_rcv_wnd;
 	u32			  tw_ts_offset;
 	u32			  tw_ts_recent;
diff --git a/include/net/sock.h b/include/net/sock.h
index 6efba5f998f1..d43b2c0cbee4 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -227,6 +227,7 @@ struct sock_common {
 	union {
 		int		skc_incoming_cpu;
 		u32		skc_rcv_wnd;
+		u32		skc_tw_rcv_nxt; /* struct tcp_timewait_sock  */
 	};
 
 	atomic_t		skc_refcnt;
@@ -235,6 +236,7 @@ struct sock_common {
 	union {
 		u32		skc_rxhash;
 		u32		skc_window_clamp;
+		u32		skc_tw_snd_nxt; /* struct tcp_timewait_sock */
 	};
 	/* public: */
 };
-- 
2.6.0.rc2.230.g3dd15c0

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Tom Herbert]

On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> SO_INCOMING_CPU as added in commit 2c8c56e15df3 was a getsockopt() command
> to fetch incoming cpu handling a particular TCP flow after accept()
>
> This commits adds setsockopt() support and extends SO_REUSEPORT selection
> logic : If a TCP listener or UDP socket has this option set, a packet is
> delivered to this socket only if CPU handling the packet matches the specified one.
>
> This allows to build very efficient TCP servers, using one thread per cpu,
> as the associated TCP listener should only accept flows handled in softirq
> by the same cpu. This provides optimal NUMA/SMP behavior and keep cpu caches hot.
>
Please look again at my SO_INCOMING_CPU_MASK patches to see it these
will work. I believe SO_INCOMING_CPU setsockopt is probably a subset
of that functionality. A single CPU assigned to socket forces an
application design to have one thread per CPU-- this may be overkill.
It's probably be sufficient in many cases to have just one listener
thread per NUMA node.

Thanks,
Tom

> Note that __inet_lookup_listener() still has to iterate over the list of
> all listeners. Following patch puts sk_refcnt in a different cache line
> to let this iteration hit only shared and read mostly cache lines.
>
> Signed-off-by: Eric Dumazet <edumazet@google.com>
> ---
>  include/net/sock.h          | 11 +++++------
>  net/core/sock.c             |  5 +++++
>  net/ipv4/inet_hashtables.c  |  5 +++++
>  net/ipv4/udp.c              | 12 +++++++++++-
>  net/ipv6/inet6_hashtables.c |  5 +++++
>  net/ipv6/udp.c              | 11 +++++++++++
>  6 files changed, 42 insertions(+), 7 deletions(-)
>
> diff --git a/include/net/sock.h b/include/net/sock.h
> index dfe2eb8e1132..00f60bea983b 100644
> --- a/include/net/sock.h
> +++ b/include/net/sock.h
> @@ -150,6 +150,7 @@ typedef __u64 __bitwise __addrpair;
>   *     @skc_node: main hash linkage for various protocol lookup tables
>   *     @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
>   *     @skc_tx_queue_mapping: tx queue number for this connection
> + *     @skc_incoming_cpu: record/match cpu processing incoming packets
>   *     @skc_refcnt: reference count
>   *
>   *     This is the minimal network layer representation of sockets, the header
> @@ -212,6 +213,8 @@ struct sock_common {
>                 struct hlist_nulls_node skc_nulls_node;
>         };
>         int                     skc_tx_queue_mapping;
> +       int                     skc_incoming_cpu;
> +
>         atomic_t                skc_refcnt;
>         /* private: */
>         int                     skc_dontcopy_end[0];
> @@ -274,7 +277,7 @@ struct cg_proto;
>    *    @sk_rcvtimeo: %SO_RCVTIMEO setting
>    *    @sk_sndtimeo: %SO_SNDTIMEO setting
>    *    @sk_rxhash: flow hash received from netif layer
> -  *    @sk_incoming_cpu: record cpu processing incoming packets
> +  *    @sk_incoming_cpu: record/match cpu processing incoming packets
>    *    @sk_txhash: computed flow hash for use on transmit
>    *    @sk_filter: socket filtering instructions
>    *    @sk_timer: sock cleanup timer
> @@ -331,6 +334,7 @@ struct sock {
>  #define sk_v6_daddr            __sk_common.skc_v6_daddr
>  #define sk_v6_rcv_saddr        __sk_common.skc_v6_rcv_saddr
>  #define sk_cookie              __sk_common.skc_cookie
> +#define sk_incoming_cpu                __sk_common.skc_incoming_cpu
>
>         socket_lock_t           sk_lock;
>         struct sk_buff_head     sk_receive_queue;
> @@ -353,11 +357,6 @@ struct sock {
>  #ifdef CONFIG_RPS
>         __u32                   sk_rxhash;
>  #endif
> -       u16                     sk_incoming_cpu;
> -       /* 16bit hole
> -        * Warned : sk_incoming_cpu can be set from softirq,
> -        * Do not use this hole without fully understanding possible issues.
> -        */
>
>         __u32                   sk_txhash;
>  #ifdef CONFIG_NET_RX_BUSY_POLL
> diff --git a/net/core/sock.c b/net/core/sock.c
> index 7dd1263e4c24..1071f9380250 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -988,6 +988,10 @@ set_rcvbuf:
>                                          sk->sk_max_pacing_rate);
>                 break;
>
> +       case SO_INCOMING_CPU:
> +               sk->sk_incoming_cpu = val;
> +               break;
> +
>         default:
>                 ret = -ENOPROTOOPT;
>                 break;
> @@ -2353,6 +2357,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
>
>         sk->sk_max_pacing_rate = ~0U;
>         sk->sk_pacing_rate = ~0U;
> +       sk->sk_incoming_cpu = -1;
>         /*
>          * Before updating sk_refcnt, we must commit prior changes to memory
>          * (Documentation/RCU/rculist_nulls.txt for details)
> diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
> index bed8886a4b6c..eabcfbc13afb 100644
> --- a/net/ipv4/inet_hashtables.c
> +++ b/net/ipv4/inet_hashtables.c
> @@ -185,6 +185,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                                 return -1;
>                         score += 4;
>                 }
> +               if (sk->sk_incoming_cpu != -1) {
> +                       if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                               return -1;
> +                       score++;
> +               }
>         }
>         return score;
>  }
> diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
> index e1fc129099ea..de675b796f78 100644
> --- a/net/ipv4/udp.c
> +++ b/net/ipv4/udp.c
> @@ -375,7 +375,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                         return -1;
>                 score += 4;
>         }
> -
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
>         return score;
>  }
>
> @@ -419,6 +423,12 @@ static inline int compute_score2(struct sock *sk, struct net *net,
>                 score += 4;
>         }
>
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
> +
>         return score;
>  }
>
> diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
> index 6ac8dad0138a..af3d7f826bff 100644
> --- a/net/ipv6/inet6_hashtables.c
> +++ b/net/ipv6/inet6_hashtables.c
> @@ -114,6 +114,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                                 return -1;
>                         score++;
>                 }
> +               if (sk->sk_incoming_cpu != -1) {
> +                       if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                               return -1;
> +                       score++;
> +               }
>         }
>         return score;
>  }
> diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
> index 0aba654f5b91..222fdc780405 100644
> --- a/net/ipv6/udp.c
> +++ b/net/ipv6/udp.c
> @@ -182,6 +182,12 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                 score++;
>         }
>
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
> +
>         return score;
>  }
>
> @@ -223,6 +229,11 @@ static inline int compute_score2(struct sock *sk, struct net *net,
>                 score++;
>         }
>
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
>         return score;
>  }
>
> --
> 2.6.0.rc2.230.g3dd15c0
>
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3619 bytes --]

Hi Eric,

[auto build test WARNING on net-next/master -- if it's inappropriate base, please ignore]

reproduce: make htmldocs

All warnings (new ones prefixed by >>):

   include/linux/skbuff.h:833: warning: No description found for parameter 'sk'
>> include/net/sock.h:442: warning: Excess struct/union/enum/typedef member 'sk_incoming_cpu' description in 'sock'
   net/core/gen_stats.c:155: warning: No description found for parameter 'cpu'
   net/core/gen_estimator.c:212: warning: No description found for parameter 'cpu_bstats'
   net/core/gen_estimator.c:303: warning: No description found for parameter 'cpu_bstats'
   net/core/dev.c:6102: warning: No description found for parameter 'len'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_XMIT_DST_RELEASE_PERM' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_IPVLAN_MASTER' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_IPVLAN_SLAVE' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1791: warning: No description found for parameter 'ptype_all'
   include/linux/netdevice.h:1791: warning: No description found for parameter 'ptype_specific'

vim +442 include/net/sock.h

^1da177e4 Linus Torvalds     2005-04-16  426  	int			sk_write_pending;
d5f642384 Alexey Dobriyan    2008-11-04  427  #ifdef CONFIG_SECURITY
^1da177e4 Linus Torvalds     2005-04-16  428  	void			*sk_security;
d5f642384 Alexey Dobriyan    2008-11-04  429  #endif
4a19ec580 Laszlo Attila Toth 2008-01-30  430  	__u32			sk_mark;
e181a5430 Mathias Krause     2015-07-19  431  #ifdef CONFIG_CGROUP_NET_CLASSID
f84517253 Herbert Xu         2010-05-24  432  	u32			sk_classid;
e181a5430 Mathias Krause     2015-07-19  433  #endif
e1aab161e Glauber Costa      2011-12-11  434  	struct cg_proto		*sk_cgrp;
^1da177e4 Linus Torvalds     2005-04-16  435  	void			(*sk_state_change)(struct sock *sk);
676d23690 David S. Miller    2014-04-11  436  	void			(*sk_data_ready)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  437  	void			(*sk_write_space)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  438  	void			(*sk_error_report)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  439  	int			(*sk_backlog_rcv)(struct sock *sk,
^1da177e4 Linus Torvalds     2005-04-16  440  						  struct sk_buff *skb);
^1da177e4 Linus Torvalds     2005-04-16  441  	void                    (*sk_destruct)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16 @442  };
^1da177e4 Linus Torvalds     2005-04-16  443  
559835ea7 Pravin B Shelar    2013-09-24  444  #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
559835ea7 Pravin B Shelar    2013-09-24  445  
559835ea7 Pravin B Shelar    2013-09-24  446  #define rcu_dereference_sk_user_data(sk)	rcu_dereference(__sk_user_data((sk)))
559835ea7 Pravin B Shelar    2013-09-24  447  #define rcu_assign_sk_user_data(sk, ptr)	rcu_assign_pointer(__sk_user_data((sk)), ptr)
559835ea7 Pravin B Shelar    2013-09-24  448  
4a17fd522 Pavel Emelyanov    2012-04-19  449  /*
4a17fd522 Pavel Emelyanov    2012-04-19  450   * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK

:::::: The code at line 442 was first introduced by commit
:::::: 1da177e4c3f41524e886b7f1b8a0c1fc7321cac2 Linux-2.6.12-rc2

:::::: TO: Linus Torvalds <torvalds@ppc970.osdl.org>
:::::: CC: Linus Torvalds <torvalds@ppc970.osdl.org>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 6062 bytes --]

Re: [PATCH net-next 2/4] net: align sk_refcnt on 128 bytes boundary

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3728 bytes --]

Hi Eric,

[auto build test WARNING on net-next/master -- if it's inappropriate base, please ignore]

reproduce: make htmldocs

All warnings (new ones prefixed by >>):

   include/linux/skbuff.h:833: warning: No description found for parameter 'sk'
>> include/net/sock.h:453: warning: Excess struct/union/enum/typedef member 'sk_flags' description in 'sock'
   include/net/sock.h:453: warning: Excess struct/union/enum/typedef member 'sk_incoming_cpu' description in 'sock'
   net/core/gen_stats.c:155: warning: No description found for parameter 'cpu'
   net/core/gen_estimator.c:212: warning: No description found for parameter 'cpu_bstats'
   net/core/gen_estimator.c:303: warning: No description found for parameter 'cpu_bstats'
   net/core/dev.c:6102: warning: No description found for parameter 'len'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_XMIT_DST_RELEASE_PERM' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_IPVLAN_MASTER' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_IPVLAN_SLAVE' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1791: warning: No description found for parameter 'ptype_all'
   include/linux/netdevice.h:1791: warning: No description found for parameter 'ptype_specific'

vim +453 include/net/sock.h

^1da177e4 Linus Torvalds     2005-04-16  437  	int			sk_write_pending;
d5f642384 Alexey Dobriyan    2008-11-04  438  #ifdef CONFIG_SECURITY
^1da177e4 Linus Torvalds     2005-04-16  439  	void			*sk_security;
d5f642384 Alexey Dobriyan    2008-11-04  440  #endif
4a19ec580 Laszlo Attila Toth 2008-01-30  441  	__u32			sk_mark;
e181a5430 Mathias Krause     2015-07-19  442  #ifdef CONFIG_CGROUP_NET_CLASSID
f84517253 Herbert Xu         2010-05-24  443  	u32			sk_classid;
e181a5430 Mathias Krause     2015-07-19  444  #endif
e1aab161e Glauber Costa      2011-12-11  445  	struct cg_proto		*sk_cgrp;
^1da177e4 Linus Torvalds     2005-04-16  446  	void			(*sk_state_change)(struct sock *sk);
676d23690 David S. Miller    2014-04-11  447  	void			(*sk_data_ready)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  448  	void			(*sk_write_space)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  449  	void			(*sk_error_report)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  450  	int			(*sk_backlog_rcv)(struct sock *sk,
^1da177e4 Linus Torvalds     2005-04-16  451  						  struct sk_buff *skb);
^1da177e4 Linus Torvalds     2005-04-16  452  	void                    (*sk_destruct)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16 @453  };
^1da177e4 Linus Torvalds     2005-04-16  454  
559835ea7 Pravin B Shelar    2013-09-24  455  #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
559835ea7 Pravin B Shelar    2013-09-24  456  
559835ea7 Pravin B Shelar    2013-09-24  457  #define rcu_dereference_sk_user_data(sk)	rcu_dereference(__sk_user_data((sk)))
559835ea7 Pravin B Shelar    2013-09-24  458  #define rcu_assign_sk_user_data(sk, ptr)	rcu_assign_pointer(__sk_user_data((sk)), ptr)
559835ea7 Pravin B Shelar    2013-09-24  459  
4a17fd522 Pavel Emelyanov    2012-04-19  460  /*
4a17fd522 Pavel Emelyanov    2012-04-19  461   * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK

:::::: The code at line 453 was first introduced by commit
:::::: 1da177e4c3f41524e886b7f1b8a0c1fc7321cac2 Linux-2.6.12-rc2

:::::: TO: Linus Torvalds <torvalds@ppc970.osdl.org>
:::::: CC: Linus Torvalds <torvalds@ppc970.osdl.org>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 6062 bytes --]

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Eric Dumazet]

On Thu, 2015-10-08 at 09:03 -0700, Tom Herbert wrote:
> On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> > SO_INCOMING_CPU as added in commit 2c8c56e15df3 was a getsockopt() command
> > to fetch incoming cpu handling a particular TCP flow after accept()
> >
> > This commits adds setsockopt() support and extends SO_REUSEPORT selection
> > logic : If a TCP listener or UDP socket has this option set, a packet is
> > delivered to this socket only if CPU handling the packet matches the specified one.
> >
> > This allows to build very efficient TCP servers, using one thread per cpu,
> > as the associated TCP listener should only accept flows handled in softirq
> > by the same cpu. This provides optimal NUMA/SMP behavior and keep cpu caches hot.
> >
> Please look again at my SO_INCOMING_CPU_MASK patches to see it these
> will work. I believe SO_INCOMING_CPU setsockopt is probably a subset
> of that functionality. A single CPU assigned to socket forces an
> application design to have one thread per CPU-- this may be overkill.
> It's probably be sufficient in many cases to have just one listener
> thread per NUMA node.


I think you misunderstood my patch.

For optimal behavior against DDOS, you need one TCP _listener_ per RX
queue on the NIC.

Not one listener per cpu !

You can then have multiple threads servicing requests coming for a
particular listener. Or one thread (cpu) servicing requests coming from
multiple TCP listeners. It's as you said an application choice/design.

Say you have 32 cpu (hyper threads) and 8 RX queues, you can create
groups of 4 threads per RX queue. Or one thread servicing all accept()

If you use 2 listeners (one per NUMA node), performance is not so good,
and SO_INCOMING_CPU_MASK adds an extra indirection, scaling poorly with
say 64 listeners.

Ying Cai has a patch to add an array selection (instead of having to
parse the list, yet to be rebased) 

I am afraid your choice of SO_INCOMING_CPU_MASK might have been driven
by the necessity of keeping the list short, but this is about to be a
non issue.

Thanks.

Re: [PATCH net-next 3/4] net: shrink struct sock and request_sock by 8 bytes

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3838 bytes --]

Hi Eric,

[auto build test WARNING on net-next/master -- if it's inappropriate base, please ignore]

reproduce: make htmldocs

All warnings (new ones prefixed by >>):

   include/linux/skbuff.h:833: warning: No description found for parameter 'sk'
   include/net/sock.h:458: warning: Excess struct/union/enum/typedef member 'sk_flags' description in 'sock'
>> include/net/sock.h:458: warning: Excess struct/union/enum/typedef member 'sk_rxhash' description in 'sock'
   include/net/sock.h:458: warning: Excess struct/union/enum/typedef member 'sk_incoming_cpu' description in 'sock'
   net/core/gen_stats.c:155: warning: No description found for parameter 'cpu'
   net/core/gen_estimator.c:212: warning: No description found for parameter 'cpu_bstats'
   net/core/gen_estimator.c:303: warning: No description found for parameter 'cpu_bstats'
   net/core/dev.c:6102: warning: No description found for parameter 'len'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_XMIT_DST_RELEASE_PERM' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_IPVLAN_MASTER' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1291: warning: Enum value 'IFF_IPVLAN_SLAVE' not described in enum 'netdev_priv_flags'
   include/linux/netdevice.h:1791: warning: No description found for parameter 'ptype_all'
   include/linux/netdevice.h:1791: warning: No description found for parameter 'ptype_specific'

vim +458 include/net/sock.h

^1da177e4 Linus Torvalds     2005-04-16  442  	int			sk_write_pending;
d5f642384 Alexey Dobriyan    2008-11-04  443  #ifdef CONFIG_SECURITY
^1da177e4 Linus Torvalds     2005-04-16  444  	void			*sk_security;
d5f642384 Alexey Dobriyan    2008-11-04  445  #endif
4a19ec580 Laszlo Attila Toth 2008-01-30  446  	__u32			sk_mark;
e181a5430 Mathias Krause     2015-07-19  447  #ifdef CONFIG_CGROUP_NET_CLASSID
f84517253 Herbert Xu         2010-05-24  448  	u32			sk_classid;
e181a5430 Mathias Krause     2015-07-19  449  #endif
e1aab161e Glauber Costa      2011-12-11  450  	struct cg_proto		*sk_cgrp;
^1da177e4 Linus Torvalds     2005-04-16  451  	void			(*sk_state_change)(struct sock *sk);
676d23690 David S. Miller    2014-04-11  452  	void			(*sk_data_ready)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  453  	void			(*sk_write_space)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  454  	void			(*sk_error_report)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16  455  	int			(*sk_backlog_rcv)(struct sock *sk,
^1da177e4 Linus Torvalds     2005-04-16  456  						  struct sk_buff *skb);
^1da177e4 Linus Torvalds     2005-04-16  457  	void                    (*sk_destruct)(struct sock *sk);
^1da177e4 Linus Torvalds     2005-04-16 @458  };
^1da177e4 Linus Torvalds     2005-04-16  459  
559835ea7 Pravin B Shelar    2013-09-24  460  #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
559835ea7 Pravin B Shelar    2013-09-24  461  
559835ea7 Pravin B Shelar    2013-09-24  462  #define rcu_dereference_sk_user_data(sk)	rcu_dereference(__sk_user_data((sk)))
559835ea7 Pravin B Shelar    2013-09-24  463  #define rcu_assign_sk_user_data(sk, ptr)	rcu_assign_pointer(__sk_user_data((sk)), ptr)
559835ea7 Pravin B Shelar    2013-09-24  464  
4a17fd522 Pavel Emelyanov    2012-04-19  465  /*
4a17fd522 Pavel Emelyanov    2012-04-19  466   * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK

:::::: The code at line 458 was first introduced by commit
:::::: 1da177e4c3f41524e886b7f1b8a0c1fc7321cac2 Linux-2.6.12-rc2

:::::: TO: Linus Torvalds <torvalds@ppc970.osdl.org>
:::::: CC: Linus Torvalds <torvalds@ppc970.osdl.org>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 6062 bytes --]

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Tom Herbert]

On Thu, Oct 8, 2015 at 9:29 AM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
> On Thu, 2015-10-08 at 09:03 -0700, Tom Herbert wrote:
>> On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
>> > SO_INCOMING_CPU as added in commit 2c8c56e15df3 was a getsockopt() command
>> > to fetch incoming cpu handling a particular TCP flow after accept()
>> >
>> > This commits adds setsockopt() support and extends SO_REUSEPORT selection
>> > logic : If a TCP listener or UDP socket has this option set, a packet is
>> > delivered to this socket only if CPU handling the packet matches the specified one.
>> >
>> > This allows to build very efficient TCP servers, using one thread per cpu,
>> > as the associated TCP listener should only accept flows handled in softirq
>> > by the same cpu. This provides optimal NUMA/SMP behavior and keep cpu caches hot.
>> >
>> Please look again at my SO_INCOMING_CPU_MASK patches to see it these
>> will work. I believe SO_INCOMING_CPU setsockopt is probably a subset
>> of that functionality. A single CPU assigned to socket forces an
>> application design to have one thread per CPU-- this may be overkill.
>> It's probably be sufficient in many cases to have just one listener
>> thread per NUMA node.
>
>
> I think you misunderstood my patch.
>
> For optimal behavior against DDOS, you need one TCP _listener_ per RX
> queue on the NIC.
>
I see. We are not using SO_INCOMING_CPU_MASK as a defense against
DDOS. It's used ensure affinity in application connection processing
between CPUs. For instance, if we have two NUMA nodes we can start two
instances of the application bound to each node and then use
SO_REUSEPORT and SO_INCOMING_CPU_MASK to ensure connections are
processed on the the same NUMA node. Packets crossing NUMA boundaries
even with RFS is painful.

Tom

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Eric Dumazet]

On Thu, 2015-10-08 at 09:29 -0700, Eric Dumazet wrote:

> For optimal behavior against DDOS, you need one TCP _listener_ per RX
> queue on the NIC.

I will reword the changelog to make this clear, and fix the htmldocs
warning reported by kbuild in V2.

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Eric Dumazet]

On Thu, 2015-10-08 at 09:44 -0700, Tom Herbert wrote:

> I see. We are not using SO_INCOMING_CPU_MASK as a defense against
> DDOS. It's used ensure affinity in application connection processing
> between CPUs. For instance, if we have two NUMA nodes we can start two
> instances of the application bound to each node and then use
> SO_REUSEPORT and SO_INCOMING_CPU_MASK to ensure connections are
> processed on the the same NUMA node. Packets crossing NUMA boundaries
> even with RFS is painful.

Then maybe you need something simpler than a mask of cpus, like
SO_INCOMING_NODE ?

Note that this could also be automatically tuned.

If you have a bunch of listeners on one port (could be 2), then the
underlying node on which TCP socket was allocated could be used as a
score modifier in compute_score()

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Tom Herbert]

On Thu, Oct 8, 2015 at 10:00 AM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
> On Thu, 2015-10-08 at 09:44 -0700, Tom Herbert wrote:
>
>> I see. We are not using SO_INCOMING_CPU_MASK as a defense against
>> DDOS. It's used ensure affinity in application connection processing
>> between CPUs. For instance, if we have two NUMA nodes we can start two
>> instances of the application bound to each node and then use
>> SO_REUSEPORT and SO_INCOMING_CPU_MASK to ensure connections are
>> processed on the the same NUMA node. Packets crossing NUMA boundaries
>> even with RFS is painful.
>
> Then maybe you need something simpler than a mask of cpus, like
> SO_INCOMING_NODE ?
>
> Note that this could also be automatically tuned.
>
> If you have a bunch of listeners on one port (could be 2), then the
> underlying node on which TCP socket was allocated could be used as a
> score modifier in compute_score()
>
Maybe, but I really like the simplicity of SO_INCOMING_CPU_MASK in
that it doesn't depend on NUMA configuration or RSS configuration.
It's very easy for the application to just set the same incoming CPU
mask to be the same CPU mask that the listener thread is bound to.
We've seen some nice performance benefits in doing with memcached,
that is to run an instances bound to the set of CPUs for each set NUMA
node and bind listener threads to the same set.

Tom

>

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Tom Herbert]

On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> SO_INCOMING_CPU as added in commit 2c8c56e15df3 was a getsockopt() command
> to fetch incoming cpu handling a particular TCP flow after accept()
>
> This commits adds setsockopt() support and extends SO_REUSEPORT selection
> logic : If a TCP listener or UDP socket has this option set, a packet is
> delivered to this socket only if CPU handling the packet matches the specified one.
>
> This allows to build very efficient TCP servers, using one thread per cpu,
> as the associated TCP listener should only accept flows handled in softirq
> by the same cpu. This provides optimal NUMA/SMP behavior and keep cpu caches hot.
>
> Note that __inet_lookup_listener() still has to iterate over the list of
> all listeners. Following patch puts sk_refcnt in a different cache line
> to let this iteration hit only shared and read mostly cache lines.
>
> Signed-off-by: Eric Dumazet <edumazet@google.com>
> ---
>  include/net/sock.h          | 11 +++++------
>  net/core/sock.c             |  5 +++++
>  net/ipv4/inet_hashtables.c  |  5 +++++
>  net/ipv4/udp.c              | 12 +++++++++++-
>  net/ipv6/inet6_hashtables.c |  5 +++++
>  net/ipv6/udp.c              | 11 +++++++++++
>  6 files changed, 42 insertions(+), 7 deletions(-)
>
> diff --git a/include/net/sock.h b/include/net/sock.h
> index dfe2eb8e1132..00f60bea983b 100644
> --- a/include/net/sock.h
> +++ b/include/net/sock.h
> @@ -150,6 +150,7 @@ typedef __u64 __bitwise __addrpair;
>   *     @skc_node: main hash linkage for various protocol lookup tables
>   *     @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
>   *     @skc_tx_queue_mapping: tx queue number for this connection
> + *     @skc_incoming_cpu: record/match cpu processing incoming packets
>   *     @skc_refcnt: reference count
>   *
>   *     This is the minimal network layer representation of sockets, the header
> @@ -212,6 +213,8 @@ struct sock_common {
>                 struct hlist_nulls_node skc_nulls_node;
>         };
>         int                     skc_tx_queue_mapping;
> +       int                     skc_incoming_cpu;
> +
>         atomic_t                skc_refcnt;
>         /* private: */
>         int                     skc_dontcopy_end[0];
> @@ -274,7 +277,7 @@ struct cg_proto;
>    *    @sk_rcvtimeo: %SO_RCVTIMEO setting
>    *    @sk_sndtimeo: %SO_SNDTIMEO setting
>    *    @sk_rxhash: flow hash received from netif layer
> -  *    @sk_incoming_cpu: record cpu processing incoming packets
> +  *    @sk_incoming_cpu: record/match cpu processing incoming packets
>    *    @sk_txhash: computed flow hash for use on transmit
>    *    @sk_filter: socket filtering instructions
>    *    @sk_timer: sock cleanup timer
> @@ -331,6 +334,7 @@ struct sock {
>  #define sk_v6_daddr            __sk_common.skc_v6_daddr
>  #define sk_v6_rcv_saddr        __sk_common.skc_v6_rcv_saddr
>  #define sk_cookie              __sk_common.skc_cookie
> +#define sk_incoming_cpu                __sk_common.skc_incoming_cpu
>
>         socket_lock_t           sk_lock;
>         struct sk_buff_head     sk_receive_queue;
> @@ -353,11 +357,6 @@ struct sock {
>  #ifdef CONFIG_RPS
>         __u32                   sk_rxhash;
>  #endif
> -       u16                     sk_incoming_cpu;
> -       /* 16bit hole
> -        * Warned : sk_incoming_cpu can be set from softirq,
> -        * Do not use this hole without fully understanding possible issues.
> -        */
>
>         __u32                   sk_txhash;
>  #ifdef CONFIG_NET_RX_BUSY_POLL
> diff --git a/net/core/sock.c b/net/core/sock.c
> index 7dd1263e4c24..1071f9380250 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -988,6 +988,10 @@ set_rcvbuf:
>                                          sk->sk_max_pacing_rate);
>                 break;
>
> +       case SO_INCOMING_CPU:
> +               sk->sk_incoming_cpu = val;
> +               break;
> +
>         default:
>                 ret = -ENOPROTOOPT;
>                 break;
> @@ -2353,6 +2357,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
>
>         sk->sk_max_pacing_rate = ~0U;
>         sk->sk_pacing_rate = ~0U;
> +       sk->sk_incoming_cpu = -1;
>         /*
>          * Before updating sk_refcnt, we must commit prior changes to memory
>          * (Documentation/RCU/rculist_nulls.txt for details)
> diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
> index bed8886a4b6c..eabcfbc13afb 100644
> --- a/net/ipv4/inet_hashtables.c
> +++ b/net/ipv4/inet_hashtables.c
> @@ -185,6 +185,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                                 return -1;
>                         score += 4;
>                 }
> +               if (sk->sk_incoming_cpu != -1) {
> +                       if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                               return -1;
> +                       score++;
> +               }

If the incoming CPU is set for a connected UDP via
sk_incoming_cpu_update, wouldn't this check subsequently _only_ allow
packets for that socket to come from the same CPU?

Also, the check seems a little austere. Why not do something like:

               if (sk->sk_incoming_cpu != -1) {
                       if (sk->sk_incoming_cpu != raw_smp_processor_id())
                            score += 4;
               }

My worry is that the packet steering configuration may change without
the application's knowledge, so it's possible packets may come in on
CPUs that the are unexpected to the application and then they would be
dropped without matching a socket. I suppose that this could work with
the original patch if a socket is bound to every CPU or there is at
least one listener socket that is not bound to any CPU.

>  }
> diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
> index e1fc129099ea..de675b796f78 100644
> --- a/net/ipv4/udp.c
> +++ b/net/ipv4/udp.c
> @@ -375,7 +375,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                         return -1;
>                 score += 4;
>         }
> -
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
>         return score;
>  }
>
> @@ -419,6 +423,12 @@ static inline int compute_score2(struct sock *sk, struct net *net,
>                 score += 4;
>         }
>
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
> +
>         return score;
>  }
>
> diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
> index 6ac8dad0138a..af3d7f826bff 100644
> --- a/net/ipv6/inet6_hashtables.c
> +++ b/net/ipv6/inet6_hashtables.c
> @@ -114,6 +114,11 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                                 return -1;
>                         score++;
>                 }
> +               if (sk->sk_incoming_cpu != -1) {
> +                       if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                               return -1;
> +                       score++;
> +               }
>         }
>         return score;
>  }
> diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
> index 0aba654f5b91..222fdc780405 100644
> --- a/net/ipv6/udp.c
> +++ b/net/ipv6/udp.c
> @@ -182,6 +182,12 @@ static inline int compute_score(struct sock *sk, struct net *net,
>                 score++;
>         }
>
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
> +
>         return score;
>  }
>
> @@ -223,6 +229,11 @@ static inline int compute_score2(struct sock *sk, struct net *net,
>                 score++;
>         }
>
> +       if (sk->sk_incoming_cpu != -1) {
> +               if (sk->sk_incoming_cpu != raw_smp_processor_id())
> +                       return -1;
> +               score++;
> +       }
>         return score;
>  }
>
> --
> 2.6.0.rc2.230.g3dd15c0
>
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH net-next 1/4] net: SO_INCOMING_CPU setsockopt() support

[Eric Dumazet]

On Thu, 2015-10-08 at 13:53 -0700, Tom Herbert wrote:

> If the incoming CPU is set for a connected UDP via
> sk_incoming_cpu_update, wouldn't this check subsequently _only_ allow
> packets for that socket to come from the same CPU?
> 

Hmm, I thought the SO_REUSEPORT path would be taken only for non
connected UDP sockets (like TCP listeners.).

But you might be right !

> Also, the check seems a little austere. Why not do something like:
> 
>                if (sk->sk_incoming_cpu != -1) {
>                        if (sk->sk_incoming_cpu != raw_smp_processor_id())
>                             score += 4;
>                }
> 
> My worry is that the packet steering configuration may change without
> the application's knowledge, so it's possible packets may come in on
> CPUs that the are unexpected to the application and then they would be
> dropped without matching a socket. I suppose that this could work with
> the original patch if a socket is bound to every CPU or there is at
> least one listener socket that is not bound to any CPU.

This is what I initially wrote, then I attempted a short cut, (abort
full list scan), then forgot to re-instate the first try, when I decided
to let this for future patch (Ying patch)

if (sk->sk_incoming_cpu == raw_smp_processor_id())
    score++;

(Note we do not even have to test for sk_incoming_cpu == -1 in this
variant)

I'll include this in v2.

Thanks.

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Tom Herbert]

On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> As promised in last patch series, we implement a better SO_REUSEPORT
> strategy, based on cpu affinities if selected by the application.
>
> We also moved sk_refcnt out of the cache line containing the lookup
> keys, as it was considerably slowing down smp operations because
> of false sharing. This was simpler than converting listen sockets
> to conventional RCU (to avoid sk_refcnt dirtying)
>
> Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
>
Is this IPv4, IPv6, or some combination of the two ? :-)

> Eric Dumazet (4):
>   net: SO_INCOMING_CPU setsockopt() support
>   net: align sk_refcnt on 128 bytes boundary
>   net: shrink struct sock and request_sock by 8 bytes
>   tcp: shrink tcp_timewait_sock by 8 bytes
>
>  include/linux/tcp.h              |  4 ++--
>  include/net/inet_timewait_sock.h |  2 +-
>  include/net/request_sock.h       |  7 +++----
>  include/net/sock.h               | 37 +++++++++++++++++++++++++++----------
>  net/core/sock.c                  |  5 +++++
>  net/ipv4/inet_hashtables.c       |  5 +++++
>  net/ipv4/syncookies.c            |  4 ++--
>  net/ipv4/tcp_input.c             |  2 +-
>  net/ipv4/tcp_ipv4.c              |  2 +-
>  net/ipv4/tcp_minisocks.c         | 18 +++++++++---------
>  net/ipv4/tcp_output.c            |  2 +-
>  net/ipv4/udp.c                   | 12 +++++++++++-
>  net/ipv6/inet6_hashtables.c      |  5 +++++
>  net/ipv6/syncookies.c            |  4 ++--
>  net/ipv6/tcp_ipv6.c              |  2 +-
>  net/ipv6/udp.c                   | 11 +++++++++++
>  16 files changed, 87 insertions(+), 35 deletions(-)
>
> --
> 2.6.0.rc2.230.g3dd15c0
>
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Eric Dumazet]

On Thu, 2015-10-08 at 20:42 -0700, Tom Herbert wrote:
> On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> > As promised in last patch series, we implement a better SO_REUSEPORT
> > strategy, based on cpu affinities if selected by the application.
> >
> > We also moved sk_refcnt out of the cache line containing the lookup
> > keys, as it was considerably slowing down smp operations because
> > of false sharing. This was simpler than converting listen sockets
> > to conventional RCU (to avoid sk_refcnt dirtying)
> >
> > Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
> >
> Is this IPv4, IPv6, or some combination of the two ? :-)

IPv4 only (mostly because I was using trafgen and its csumtcp() only
deals with IPv4 and I am lazy)

I guess IPv6 one might hit some issues before reaching TCP stack, I do
not see anything performance related in TCP itself.

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Eric Dumazet]

On Fri, 2015-10-09 at 03:50 -0700, Eric Dumazet wrote:
> On Thu, 2015-10-08 at 20:42 -0700, Tom Herbert wrote:
> > On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> > > As promised in last patch series, we implement a better SO_REUSEPORT
> > > strategy, based on cpu affinities if selected by the application.
> > >
> > > We also moved sk_refcnt out of the cache line containing the lookup
> > > keys, as it was considerably slowing down smp operations because
> > > of false sharing. This was simpler than converting listen sockets
> > > to conventional RCU (to avoid sk_refcnt dirtying)
> > >
> > > Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
> > >
> > Is this IPv4, IPv6, or some combination of the two ? :-)
> 
> IPv4 only (mostly because I was using trafgen and its csumtcp() only
> deals with IPv4 and I am lazy)
> 
> I guess IPv6 one might hit some issues before reaching TCP stack, I do
> not see anything performance related in TCP itself.
> 
> 

So the answer is : about 800,000 SYN per second in IPV6 with purely DDOS attack

We hit neighbor cache badly.

  377.188231] neighbour: ndisc_cache: neighbor table overflow!
[  377.188234] neighbour: ndisc_cache: neighbor table overflow!
[  382.193043] net_ratelimit: 36622 callbacks suppressed
[  382.193046] neighbour: ndisc_cache: neighbor table overflow!
[  382.193051] neighbour: ndisc_cache: neighbor table overflow!
[  382.193054] neighbour: ndisc_cache: neighbor table overflow!

    59.79%  [kernel]  [k] queued_spin_lock_slowpath 
     4.94%  [kernel]  [k] queued_write_lock_slowpath
     2.12%  [kernel]  [k] sha_transform             
     1.93%  [kernel]  [k] ip6_pol_route.isra.47     
     1.58%  [kernel]  [k] __neigh_create            
     1.30%  [kernel]  [k] inet6_lookup_listener     
     1.19%  [kernel]  [k] memcpy_erms               
     1.09%  [kernel]  [k] _raw_read_lock_bh         
     0.88%  [kernel]  [k] memset_erms               
     0.85%  [kernel]  [k] __inet6_lookup_established
     0.84%  [kernel]  [k] ndisc_constructor         
     0.78%  [kernel]  [k] fib6_get_table            
     0.71%  [kernel]  [k] ip6t_do_table             
     0.71%  [kernel]  [k] _raw_read_unlock_bh       
     0.66%  [kernel]  [k] _raw_write_lock_bh        
     0.62%  [kernel]  [k] fib6_lookup               
     0.54%  [kernel]  [k] tcp_make_synack           
     0.54%  [kernel]  [k] tcp_conn_request          

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Eric Dumazet]

On Fri, 2015-10-09 at 07:29 -0700, Eric Dumazet wrote:

> So the answer is : about 800,000 SYN per second in IPV6 with purely DDOS attack

My IPv6 routing setup was a bit silly ;)

With a slightly better one, we reach 3.8 Mpps and kernel profile looks
like :

    21.22%  [kernel]  [k] ip6_pol_route.isra.47      
    11.42%  [kernel]  [k] _raw_read_lock_bh          
     9.83%  [kernel]  [k] fib6_lookup                
     8.96%  [kernel]  [k] _raw_read_unlock_bh        
     8.47%  [kernel]  [k] fib6_get_table             
     4.01%  [kernel]  [k] __inet6_lookup_established 
     2.94%  [kernel]  [k] memcpy_erms                
     2.36%  [kernel]  [k] dst_release                

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Daniel Borkmann]

On 10/09/2015 12:50 PM, Eric Dumazet wrote:
> On Thu, 2015-10-08 at 20:42 -0700, Tom Herbert wrote:
>> On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
>>> As promised in last patch series, we implement a better SO_REUSEPORT
>>> strategy, based on cpu affinities if selected by the application.
>>>
>>> We also moved sk_refcnt out of the cache line containing the lookup
>>> keys, as it was considerably slowing down smp operations because
>>> of false sharing. This was simpler than converting listen sockets
>>> to conventional RCU (to avoid sk_refcnt dirtying)
>>>
>>> Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
>>>
>> Is this IPv4, IPv6, or some combination of the two ? :-)
>
> IPv4 only (mostly because I was using trafgen and its csumtcp() only
> deals with IPv4 and I am lazy)

Agreed, will fix that in trafgen. ;) Thanks!

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Eric Dumazet]

On Fri, 2015-10-09 at 20:02 +0200, Daniel Borkmann wrote:

> Agreed, will fix that in trafgen. ;) Thanks!

Nice ! Thanks Daniel !

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Tobias Klauser]

On 2015-10-09 at 20:02:47 +0200, Daniel Borkmann <daniel@iogearbox.net> wrote:
> On 10/09/2015 12:50 PM, Eric Dumazet wrote:
> >On Thu, 2015-10-08 at 20:42 -0700, Tom Herbert wrote:
> >>On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> >>>As promised in last patch series, we implement a better SO_REUSEPORT
> >>>strategy, based on cpu affinities if selected by the application.
> >>>
> >>>We also moved sk_refcnt out of the cache line containing the lookup
> >>>keys, as it was considerably slowing down smp operations because
> >>>of false sharing. This was simpler than converting listen sockets
> >>>to conventional RCU (to avoid sk_refcnt dirtying)
> >>>
> >>>Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
> >>>
> >>Is this IPv4, IPv6, or some combination of the two ? :-)
> >
> >IPv4 only (mostly because I was using trafgen and its csumtcp() only
> >deals with IPv4 and I am lazy)
> 
> Agreed, will fix that in trafgen. ;) Thanks!

Daniel, I'd have som preliminary patches for trafgen ready to introduce
the csumudp6 and csumtcp6 trafgen helper functions (UDP/TCP checksum
using IPv6 pseudo headers). If you want I can push them after some
further testing later today.

Eric, would these to helpers be sufficient for your use case or do you
need any additional trafgen functionality?

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Daniel Borkmann]

On 10/13/2015 11:22 AM, Tobias Klauser wrote:
> On 2015-10-09 at 20:02:47 +0200, Daniel Borkmann <daniel@iogearbox.net> wrote:
>> On 10/09/2015 12:50 PM, Eric Dumazet wrote:
>>> On Thu, 2015-10-08 at 20:42 -0700, Tom Herbert wrote:
>>>> On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
>>>>> As promised in last patch series, we implement a better SO_REUSEPORT
>>>>> strategy, based on cpu affinities if selected by the application.
>>>>>
>>>>> We also moved sk_refcnt out of the cache line containing the lookup
>>>>> keys, as it was considerably slowing down smp operations because
>>>>> of false sharing. This was simpler than converting listen sockets
>>>>> to conventional RCU (to avoid sk_refcnt dirtying)
>>>>>
>>>>> Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
>>>>>
>>>> Is this IPv4, IPv6, or some combination of the two ? :-)
>>>
>>> IPv4 only (mostly because I was using trafgen and its csumtcp() only
>>> deals with IPv4 and I am lazy)
>>
>> Agreed, will fix that in trafgen. ;) Thanks!
>
> Daniel, I'd have som preliminary patches for trafgen ready to introduce
> the csumudp6 and csumtcp6 trafgen helper functions (UDP/TCP checksum
> using IPv6 pseudo headers). If you want I can push them after some
> further testing later today.

Was off yesterday, so looks like you beat me to it. So, yes, perfect,
please do! :)

> Eric, would these to helpers be sufficient for your use case or do you
> need any additional trafgen functionality?

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Tobias Klauser]

On 2015-10-13 at 11:28:13 +0200, Daniel Borkmann <daniel@iogearbox.net> wrote:
> On 10/13/2015 11:22 AM, Tobias Klauser wrote:
> >On 2015-10-09 at 20:02:47 +0200, Daniel Borkmann <daniel@iogearbox.net> wrote:
> >>On 10/09/2015 12:50 PM, Eric Dumazet wrote:
> >>>On Thu, 2015-10-08 at 20:42 -0700, Tom Herbert wrote:
> >>>>On Thu, Oct 8, 2015 at 8:37 AM, Eric Dumazet <edumazet@google.com> wrote:
> >>>>>As promised in last patch series, we implement a better SO_REUSEPORT
> >>>>>strategy, based on cpu affinities if selected by the application.
> >>>>>
> >>>>>We also moved sk_refcnt out of the cache line containing the lookup
> >>>>>keys, as it was considerably slowing down smp operations because
> >>>>>of false sharing. This was simpler than converting listen sockets
> >>>>>to conventional RCU (to avoid sk_refcnt dirtying)
> >>>>>
> >>>>>Could process 6.0 Mpps SYN instead of 4.2 Mpps on my test server.
> >>>>>
> >>>>Is this IPv4, IPv6, or some combination of the two ? :-)
> >>>
> >>>IPv4 only (mostly because I was using trafgen and its csumtcp() only
> >>>deals with IPv4 and I am lazy)
> >>
> >>Agreed, will fix that in trafgen. ;) Thanks!
> >
> >Daniel, I'd have som preliminary patches for trafgen ready to introduce
> >the csumudp6 and csumtcp6 trafgen helper functions (UDP/TCP checksum
> >using IPv6 pseudo headers). If you want I can push them after some
> >further testing later today.
> 
> Was off yesterday, so looks like you beat me to it. So, yes, perfect,
> please do! :)

csumudp6 and csumtcp6 for trafgen are now availble in the netsniff-ng
git as of commit 19c15748 ("trafgen: Add checksum helpers for TCP/UDP
over IPv6").

Re: [PATCH net-next 0/4] tcp: better smp listener behavior

[Eric Dumazet]

On Tue, 2015-10-13 at 12:01 +0200, Tobias Klauser wrote:

> csumudp6 and csumtcp6 for trafgen are now availble in the netsniff-ng
> git as of commit 19c15748 ("trafgen: Add checksum helpers for TCP/UDP
> over IPv6").

Awesome, I tested it and it is working properly.

Thanks Tobias !