[PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[Kuniyuki Iwashima]

Currently we fail to bind sockets to ephemeral ports when all of the ports
are exhausted even if all sockets have SO_REUSEADDR enabled. In this case,
we still have a chance to connect to the different remote hosts.

The second and third patches fix the behaviour to fully utilize all space
of the local (addr, port) tuples.

Kuniyuki Iwashima (3):
  tcp: Remove unnecessary conditions in inet_csk_bind_conflict().
  tcp: bind(addr, 0) remove the SO_REUSEADDR restriction when ephemeral
    ports are exhausted.
  tcp: Prevent port hijacking when ports are exhausted

 net/ipv4/inet_connection_sock.c | 36 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 12 deletions(-)

-- 
2.17.2 (Apple Git-113)

[PATCH net-next 1/3] tcp: Remove unnecessary conditions in inet_csk_bind_conflict().

[Kuniyuki Iwashima]

When we get an ephemeral port, the relax is false, so the SO_REUSEADDR
conditions may be evaluated twice. We do not need to check the conditions
again.

Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.co.jp>
---
 net/ipv4/inet_connection_sock.c | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index a4db79b1b643..2e9549f49a82 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -146,17 +146,15 @@ static int inet_csk_bind_conflict(const struct sock *sk,
 		    (!sk->sk_bound_dev_if ||
 		     !sk2->sk_bound_dev_if ||
 		     sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
-			if ((!reuse || !sk2->sk_reuse ||
-			    sk2->sk_state == TCP_LISTEN) &&
-			    (!reuseport || !sk2->sk_reuseport ||
-			     rcu_access_pointer(sk->sk_reuseport_cb) ||
-			     (sk2->sk_state != TCP_TIME_WAIT &&
-			     !uid_eq(uid, sock_i_uid(sk2))))) {
-				if (inet_rcv_saddr_equal(sk, sk2, true))
-					break;
-			}
-			if (!relax && reuse && sk2->sk_reuse &&
+			if (reuse && sk2->sk_reuse &&
 			    sk2->sk_state != TCP_LISTEN) {
+				if (!relax &&
+				    inet_rcv_saddr_equal(sk, sk2, true))
+					break;
+			} else if (!reuseport || !sk2->sk_reuseport ||
+				   rcu_access_pointer(sk->sk_reuseport_cb) ||
+				   (sk2->sk_state != TCP_TIME_WAIT &&
+				    !uid_eq(uid, sock_i_uid(sk2)))) {
 				if (inet_rcv_saddr_equal(sk, sk2, true))
 					break;
 			}
-- 
2.17.2 (Apple Git-113)

[PATCH net-next 2/3] tcp: bind(addr, 0) remove the SO_REUSEADDR restriction when ephemeral ports are exhausted.

[Kuniyuki Iwashima]

Commit aacd9289af8b82f5fb01bcdd53d0e3406d1333c7 ("tcp: bind() use stronger
condition for bind_conflict") introduced a restriction to forbid to bind
SO_REUSEADDR enabled sockets to the same (addr, port) tuple in order to
assign ports dispersedly so that we can connect to the same remote host.

The change results in accelerating port depletion so that we fail to bind
sockets to the same local port even if we want to connect to the different
remote hosts.

You can reproduce this issue by following instructions below.
  1. # sysctl -w net.ipv4.ip_local_port_range="32768 32768"
  2. set SO_REUSEADDR to two sockets.
  3. bind two sockets to (localhost, 0) and the latter fails.

Therefore, when ephemeral ports are exhausted, bind(addr, 0) should
fallback to the legacy behaviour to enable the SO_REUSEADDR option and make
it possible to connect to different remote (addr, port) tuples.

This patch allows us to bind SO_REUSEADDR enabled sockets to the same
(addr, port) only when all ephemeral ports are exhausted.

The only notable thing is that if all sockets bound to the same port have
both SO_REUSEADDR and SO_REUSEPORT enabled, we can bind sockets to an
ephemeral port and also do listen().

Fixes: aacd9289af8b ("tcp: bind() use stronger condition for bind_conflict")

Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.co.jp>
---
 net/ipv4/inet_connection_sock.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index 2e9549f49a82..cddeab240ea6 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -174,12 +174,14 @@ inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *
 	int port = 0;
 	struct inet_bind_hashbucket *head;
 	struct net *net = sock_net(sk);
+	bool relax = false;
 	int i, low, high, attempt_half;
 	struct inet_bind_bucket *tb;
 	u32 remaining, offset;
 	int l3mdev;
 
 	l3mdev = inet_sk_bound_l3mdev(sk);
+ports_exhausted:
 	attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0;
 other_half_scan:
 	inet_get_local_port_range(net, &low, &high);
@@ -217,7 +219,7 @@ inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *
 		inet_bind_bucket_for_each(tb, &head->chain)
 			if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
 			    tb->port == port) {
-				if (!inet_csk_bind_conflict(sk, tb, false, false))
+				if (!inet_csk_bind_conflict(sk, tb, relax, false))
 					goto success;
 				goto next_port;
 			}
@@ -237,6 +239,12 @@ inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *
 		attempt_half = 2;
 		goto other_half_scan;
 	}
+
+	if (!relax) {
+		/* We still have a chance to connect to different destinations */
+		relax = true;
+		goto ports_exhausted;
+	}
 	return NULL;
 success:
 	*port_ret = port;
-- 
2.17.2 (Apple Git-113)

[PATCH net-next 3/3] tcp: Prevent port hijacking when ports are exhausted.

[Kuniyuki Iwashima]

If all of the sockets bound to the same port have SO_REUSEADDR and
SO_REUSEPORT enabled, any other user can hijack the port by exhausting all
ephemeral ports, binding sockets to (addr, 0) and calling listen().

In this case, we should be able to bind sockets to the same port only if
the user has the first listening socket on the port or if the existing
socket is in TIME_WAIT.

Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.co.jp>
---
 net/ipv4/inet_connection_sock.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index cddeab240ea6..a6d9ee19baeb 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -131,7 +131,7 @@ static int inet_csk_bind_conflict(const struct sock *sk,
 {
 	struct sock *sk2;
 	bool reuse = sk->sk_reuse;
-	bool reuseport = !!sk->sk_reuseport && reuseport_ok;
+	bool reuseport = !!sk->sk_reuseport;
 	kuid_t uid = sock_i_uid((struct sock *)sk);
 
 	/*
@@ -148,10 +148,16 @@ static int inet_csk_bind_conflict(const struct sock *sk,
 		     sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
 			if (reuse && sk2->sk_reuse &&
 			    sk2->sk_state != TCP_LISTEN) {
-				if (!relax &&
+				if ((!relax ||
+				     (!reuseport_ok &&
+				      reuseport && sk2->sk_reuseport &&
+				      !rcu_access_pointer(sk->sk_reuseport_cb) &&
+				      (sk2->sk_state != TCP_TIME_WAIT &&
+				       !uid_eq(uid, sock_i_uid(sk2))))) &&
 				    inet_rcv_saddr_equal(sk, sk2, true))
 					break;
-			} else if (!reuseport || !sk2->sk_reuseport ||
+			} else if (!reuseport_ok ||
+				   !reuseport || !sk2->sk_reuseport ||
 				   rcu_access_pointer(sk->sk_reuseport_cb) ||
 				   (sk2->sk_state != TCP_TIME_WAIT &&
 				    !uid_eq(uid, sock_i_uid(sk2)))) {
-- 
2.17.2 (Apple Git-113)

RE: [PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[David Laight]

From: Kuniyuki Iwashima
> Sent: 20 February 2020 15:20
> 
> Currently we fail to bind sockets to ephemeral ports when all of the ports
> are exhausted even if all sockets have SO_REUSEADDR enabled. In this case,
> we still have a chance to connect to the different remote hosts.
> 
> The second and third patches fix the behaviour to fully utilize all space
> of the local (addr, port) tuples.

Would it make sense to only do this for the implicit bind() done
when connect() is called on an unbound socket?
In that case only the quadruplet of the local and remote addresses
needs to be unique.

	David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK
Registration No: 1397386 (Wales)

Re: [PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[Eric Dumazet]

On Thu, Feb 20, 2020 at 7:20 AM Kuniyuki Iwashima <kuniyu@amazon.co.jp> wrote:
>
> Currently we fail to bind sockets to ephemeral ports when all of the ports
> are exhausted even if all sockets have SO_REUSEADDR enabled. In this case,
> we still have a chance to connect to the different remote hosts.
>
> The second and third patches fix the behaviour to fully utilize all space
> of the local (addr, port) tuples.
>
> Kuniyuki Iwashima (3):
>   tcp: Remove unnecessary conditions in inet_csk_bind_conflict().
>   tcp: bind(addr, 0) remove the SO_REUSEADDR restriction when ephemeral
>     ports are exhausted.
>   tcp: Prevent port hijacking when ports are exhausted
>
>  net/ipv4/inet_connection_sock.c | 36 ++++++++++++++++++++++-----------
>  1 file changed, 24 insertions(+), 12 deletions(-)

I am travelling at the moment, so I can not really look at these
patches with enough time.

I would appreciate it if you provide tests to demonstrate your patches are safe.

Thanks.

Re: [PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[Kuniyuki Iwashima]

From: David Laight <David.Laight@ACULAB.COM>
Date: Thu, 20 Feb 2020 17:11:46 +0000
> From: Kuniyuki Iwashima
> > Sent: 20 February 2020 15:20
> >
> > Currently we fail to bind sockets to ephemeral ports when all of the ports
> > are exhausted even if all sockets have SO_REUSEADDR enabled. In this case,
> > we still have a chance to connect to the different remote hosts.
> >
> > The second and third patches fix the behaviour to fully utilize all space
> > of the local (addr, port) tuples.
> 
> Would it make sense to only do this for the implicit bind() done
> when connect() is called on an unbound socket?
> In that case only the quadruplet of the local and remote addresses
> needs to be unique.

The function to reserve a epehemral port is different between bind() and
connect(). 

  bind    : inet_csk_find_open_port
  connect : __inet_hash_connect

The connect() cannot use ports which are consumed by bind()
because __inet_hash_connect() fails to get a port if tb->fastreuse or
or tb->fastreuseport is not -1, which only __inet_hash_connect() sets.
On the other hand, bind() can use ports which are used by connect().

Moreover, we can call bind() before connect() to decide which IP to use.
By setting IP_BIND_ADDRESS_NO_PORT to socket, we can defer getting a port
until connect() is called. However, this means that getting port
is done by __inet_hash_connect, so that connect() may fail to get a local 
port if it is reserved by bind(). So if we want to reuse ports consumed by
bind(), we have to call bind() to get ports.

Without this patch, we may fail to get a ephemeral port and to fail to 
bind() in such case we should be able to reuse a local port when connecting
to remote hosts.

RE: [PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[David Laight]

From: Kuniyuki Iwashima
> Sent: 21 February 2020 10:02
> 
> From: David Laight <David.Laight@ACULAB.COM>
> Date: Thu, 20 Feb 2020 17:11:46 +0000
> > From: Kuniyuki Iwashima
> > > Sent: 20 February 2020 15:20
> > >
> > > Currently we fail to bind sockets to ephemeral ports when all of the ports
> > > are exhausted even if all sockets have SO_REUSEADDR enabled. In this case,
> > > we still have a chance to connect to the different remote hosts.
> > >
> > > The second and third patches fix the behaviour to fully utilize all space
> > > of the local (addr, port) tuples.
> >
> > Would it make sense to only do this for the implicit bind() done
> > when connect() is called on an unbound socket?
> > In that case only the quadruplet of the local and remote addresses
> > needs to be unique.
> 
> The function to reserve a epehemral port is different between bind() and
> connect().
> 
>   bind    : inet_csk_find_open_port
>   connect : __inet_hash_connect
> 
> The connect() cannot use ports which are consumed by bind()
> because __inet_hash_connect() fails to get a port if tb->fastreuse or
> or tb->fastreuseport is not -1, which only __inet_hash_connect() sets.
> On the other hand, bind() can use ports which are used by connect().

Fixing that asymmetry may make more sense.
Since the final state can already exist.
No need for SO_REUSADDR to be checked at all.

> Moreover, we can call bind() before connect() to decide which IP to use.
> By setting IP_BIND_ADDRESS_NO_PORT to socket, we can defer getting a port
> until connect() is called. However, this means that getting port
> is done by __inet_hash_connect, so that connect() may fail to get a local
> port if it is reserved by bind(). So if we want to reuse ports consumed by
> bind(), we have to call bind() to get ports.
> 
> Without this patch, we may fail to get a ephemeral port and to fail to
> bind() in such case we should be able to reuse a local port when connecting
> to remote hosts.

I suspect that opens some security holes.
There is nothing to stop you trying to call listen() (etc).

SO_REUSADDR is pretty limited (for good reason).
I normally set it on listening sockets so that the daemon can be
restarted while old connections are lurking (usually in a FIN_WAIT state).
But even that probably opens some 'holes'.

You really don't want to be allocating an ephemeral port before connect().
I'm pretty sure you can't 'unconnect' a socket leaving is bound (and usable).

	David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK
Registration No: 1397386 (Wales)

Re: [PATCH net-next 0/3] ImpArove bind(addr, 0) behaviour.

[Kuniyuki Iwashima]

From:   Eric Dumazet <edumazet@google.com>
Date:   Thu, 20 Feb 2020 11:58:00 -0800
> On Thu, Feb 20, 2020 at 7:20 AM Kuniyuki Iwashima <kuniyu@amazon.co.jp> wrote:
> >
> > Currently we fail to bind sockets to ephemeral ports when all of the ports
> > are exhausted even if all sockets have SO_REUSEADDR enabled. In this case,
> > we still have a chance to connect to the different remote hosts.
> >
> > The second and third patches fix the behaviour to fully utilize all space
> > of the local (addr, port) tuples.
> >
> > Kuniyuki Iwashima (3):
> >   tcp: Remove unnecessary conditions in inet_csk_bind_conflict().
> >   tcp: bind(addr, 0) remove the SO_REUSEADDR restriction when ephemeral
> >     ports are exhausted.
> >   tcp: Prevent port hijacking when ports are exhausted
> >
> >  net/ipv4/inet_connection_sock.c | 36 ++++++++++++++++++++++-----------
> >  1 file changed, 24 insertions(+), 12 deletions(-)
> 
> I am travelling at the moment, so I can not really look at these
> patches with enough time.
> 
> I would appreciate it if you provide tests to demonstrate your patches are safe.
> 
> Thanks.

I wrote a program below and run without patches and with patches.
Without patches, we cannot reuse ports in any pattern. With patches, we can
reuse ports if all of the socket have SO_REUSEADDR enabled and the first
socket is not in TCP_LISTEN.

So, I am sorry that the description of my third patch is wrong.

>     In this case, we should be able to bind sockets to the same port only if
>     the user has the first listening socket on the port

Also, I succeeded to reuse ports if both sockets are in TCP_CLOSE and have
SO_REUSEADDR and SO_REUSEPORT enabled, and I succeeded to call listen for
both sockets. I think only this case is not safe, so let me check the
condition.

Thank you.

=====
#include <arpa/inet.h>
#include <errno.h>
#include <netinet/in.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>


char *str[] = {
	"both",
	"sk1",
	"sk2",
	"none",
};

#define render_opt(x, i) (x[i] && x[i+1] ? str[0] :	\
			  (x[i] ? str[1] :		\
			   (x[i+1]? str[2] : str[3])))

int main(void) {
	struct sockaddr_in local_addr;	
	char addr[] = "10.0.2.15";
	char ip_str[16];
	unsigned int port;
	int fd[2];
	int len = sizeof(local_addr);
	int error = 0;
	int reuseaddr, reuseport;
	int opts[16][4] = { /* SO_REUSEADDR 1,2  SO_REUSEPORT 1,2 */
		{0,0,0,0},
		{0,0,0,1},
		{0,0,1,0},
		{0,0,1,1},

		{0,1,0,0},
		{0,1,0,1},
		{0,1,1,0},
		{0,1,1,1},

		{1,0,0,0},
		{1,0,0,1},
		{1,0,1,0},
		{1,0,1,1},

		{1,1,0,0},
		{1,1,0,1},
		{1,1,1,0},
		{1,1,1,1}
	};
	int listen_opt[4][2] = {
		{0,0},
		{1,0},
		{0,1},
		{1,1}
	};

	printf("SO_REUSEADDR\tSO_REUSEPORT\tLISTEN\tADDR1\t\t\tADDR2\t\t\tRESULT\n");

	for (int i = 0; i < 16; i++) {
		for (int l = 0; l < 4; l++) {
			printf("%s\t\t%s\t\t%s\t",
			       render_opt(opts[i], 0),
			       render_opt(opts[i], 2),
			       render_opt(listen_opt[l], 0));

			for (int j = 0; j < 2; j++) {
				fd[j] = socket(AF_INET, SOCK_STREAM, 0);

				reuseaddr = opts[i][j];
				reuseport = opts[i][j + 2];
				setsockopt(fd[j], SOL_SOCKET, SO_REUSEADDR, &reuseaddr, sizeof(int));
				setsockopt(fd[j], SOL_SOCKET, SO_REUSEPORT, &reuseport, sizeof(int));

				memset(&local_addr, 0, sizeof(local_addr));
				local_addr.sin_family = AF_INET;
				local_addr.sin_addr.s_addr = inet_addr(addr);
				local_addr.sin_port = 0;

				error = bind(fd[j], (struct sockaddr *)&local_addr, len);

				memset(&local_addr, 0, sizeof(local_addr));
				getsockname(fd[j], (struct sockaddr *)&local_addr, &len);
				inet_ntop(AF_INET, &local_addr.sin_addr, ip_str, sizeof(ip_str));
				port = ntohs(local_addr.sin_port);

				printf("%s:%u\t\t", ip_str, port);

				if (error < 0)
					goto error;

				if (listen_opt[l][j] == 1) {
					error = listen(fd[j], 5);
					if (error < 0)
						goto error;
				}
			}

			printf("o\n");
			goto close;
		error:
			printf("x\t%d: %s\n", errno, strerror(errno));
		close:
			close(fd[0]);
			close(fd[1]);
		}
	}

	return 0;
}
=====


===Without patches===
# sysctl -w net.ipv4.ip_local_port_range="32768 32768"
[    9.830967] ip_local_port_range: prefer different parity for start/end values.
net.ipv4.ip_local_port_range = 32768 32768
# ./reuse
SO_REUSEADDR	SO_REUSEPORT	LISTEN	ADDR1			ADDR2			RESULT
none		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
=====================


===With patches===
# sysctl -w net.ipv4.ip_local_port_range="32768 32768"
[   21.732038] ip_local_port_range: prefer different parity for start/end values.
net.ipv4.ip_local_port_range = 32768 32768
# ./reuse
SO_REUSEADDR	SO_REUSEPORT	LISTEN	ADDR1			ADDR2			RESULT
none		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
none		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk2		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
sk1		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		none	10.0.2.15:32768		10.0.2.15:32768		o
both		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		none	10.0.2.15:32768		10.0.2.15:32768		o
both		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		none	10.0.2.15:32768		10.0.2.15:32768		o
both		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		none	10.0.2.15:32768		10.0.2.15:32768		o
both		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
==================

Re: [PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[Kuniyuki Iwashima]

From:   Kuniyuki Iwashima <kuniyu@amazon.co.jp>
Date:   Sat, 22 Feb 2020 05:35:22 +0900

> I wrote a program below and run without patches and with patches.
> Without patches, we cannot reuse ports in any pattern. With patches, we can
> reuse ports if all of the socket have SO_REUSEADDR enabled and the first
> socket is not in TCP_LISTEN.
> 
> So, I am sorry that the description of my third patch is wrong.
> 
> >     In this case, we should be able to bind sockets to the same port only if
> >     the user has the first listening socket on the port
> 
> Also, I succeeded to reuse ports if both sockets are in TCP_CLOSE and have
> SO_REUSEADDR and SO_REUSEPORT enabled, and I succeeded to call listen for
> both sockets. I think only this case is not safe, so let me check the
> condition.

I changed the condition and it works safely. (I made a mistake when
converting the condition following De Morgan's law.)

```
diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index a6d9ee19baeb..d27ed5fe7147 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -152,8 +152,8 @@ static int inet_csk_bind_conflict(const struct sock *sk,
                                     (!reuseport_ok &&
                                      reuseport && sk2->sk_reuseport &&
                                      !rcu_access_pointer(sk->sk_reuseport_cb) &&
-                                     (sk2->sk_state != TCP_TIME_WAIT &&
-                                      !uid_eq(uid, sock_i_uid(sk2))))) &&
+                                     (sk2->sk_state == TCP_TIME_WAIT ||
+                                      uid_eq(uid, sock_i_uid(sk2))))) &&
                                    inet_rcv_saddr_equal(sk, sk2, true))
                                        break;
                        } else if (!reuseport_ok ||
```

===result of test===
both		none		none	10.0.2.15:32768		10.0.2.15:32768		o
both		none		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		none		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		none		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		none	10.0.2.15:32768		10.0.2.15:32768		o
both		sk2		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk2		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		sk2		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		none	10.0.2.15:32768		10.0.2.15:32768		o
both		sk1		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		sk1		sk2	10.0.2.15:32768		10.0.2.15:32768		o
both		sk1		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		none	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		sk1	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		sk2	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
both		both		both	10.0.2.15:32768		0.0.0.0:0		x	98: Address already in use
====================


I also tested with two users. I am sorry about having tested in python,
I will rewrite it in C later.

Both of user-a and user-b can get the same port, but one of them failed to
call listen().

So, I think this patch is safe.

===user-a gets a port===
# su user-a
# whoami
user-a
# python3
...
>>> import socket
>>> sk1 = socket.socket()
>>> sk1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
>>> sk1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
>>> sk1.bind(('10.0.2.15', 0))
>>> sk1
<socket.socket fd=3, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('10.0.2.15', 32768)>
========================

===user-b gets a port===
# su user-b
# whoami
user-b
# python3
...
>>> import socket
>>> sk2 = socket.socket()
>>> sk2.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
>>> sk2.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
>>> sk2.bind(('10.0.2.15', 0))
>>> sk2
<socket.socket fd=3, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('10.0.2.15', 32768)>
========================


===user-a listen()===
>>> sk1.listen(5)
>>> sk1
<socket.socket fd=3, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('10.0.2.15', 32768)>
=====================


===user-b listen() (failure)===
>>> sk2.listen(5)    
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
OSError: [Errno 98] Address already in use
===============================

Re: [PATCH net-next 0/3] Improve bind(addr, 0) behaviour.

[Kuniyuki Iwashima]

From:   Kuniyuki Iwashima <kuniyu@amazon.co.jp>
Date:   Sat, 22 Feb 2020 10:07:49 +0900
> I also tested with two users. I am sorry about having tested in python,
> I will rewrite it in C later.
> 
> Both of user-a and user-b can get the same port, but one of them failed to
> call listen().

I wrote a test in C and the result was the same.
If all of the sockets bound to the same port have SO_REUSEADDR and
SO_REUSEPORT enabled, two users can bind(), but can only one user listen().

If you would think these patches are safe, I'll respin the patches with
the correct conditon of the 3rd patch.

Thanks.

=====
#include <arpa/inet.h>
#include <errno.h>
#include <netinet/in.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <unistd.h>


int main(void) {
	struct sockaddr_in local_addr;
	uid_t euid[2] = {1001, 1002};
	int fd[2], error, i, port;
	int len = sizeof(local_addr);
	int reuseaddr = 1, reuseport = 1;
	char ip_str[16];

	for (i = 0; i < 2; i++) {
		if (seteuid(euid[i]) != 0)
			goto error;

		fd[i] = socket(AF_INET, SOCK_STREAM, 0);

		setsockopt(fd[i], SOL_SOCKET, SO_REUSEADDR, &reuseaddr, sizeof(int));
		setsockopt(fd[i], SOL_SOCKET, SO_REUSEPORT, &reuseport, sizeof(int));

		local_addr.sin_family = AF_INET;
		local_addr.sin_addr.s_addr = inet_addr("10.0.2.15");
		local_addr.sin_port = 0;

		error = bind(fd[i], (struct sockaddr *)&local_addr, len);

		memset(&local_addr, 0, sizeof(local_addr));
		getsockname(fd[i], (struct sockaddr *)&local_addr, &len);
		inet_ntop(AF_INET, &local_addr.sin_addr, ip_str, sizeof(ip_str));
		port = ntohs(local_addr.sin_port);

		printf("euid: %d\tbound to %s:%u\n", euid[i], ip_str, port);

		if (error != 0)
			goto error;

		if (seteuid(0) != 0)
			goto error;
	}

	for (i = 0; i < 2; i++) {
		if (seteuid(euid[i]) != 0)
			goto error;

		error = listen(fd[i], 5);

		if (error < 0)
			printf("euid: %d\tlisten failed\n", euid[i]);
		else
			printf("euid: %d\tlisten succeeded\n", euid[i]);

		if (seteuid(0) != 0)
			goto error;		
	}

	return 0;
error:
	printf("error: %d, %s\n", errno, strerror(errno));
	return -1;
}
=====

===result===
# id user-a
uid=1001(user-a) gid=1001(user-a) groups=1001(user-a)
# id user-b
uid=1002(user-b) gid=1002(user-b) groups=1002(user-b)
# sysctl -w net.ipv4.ip_local_port_range="32768 32768"
[   30.060036] ip_local_port_range: prefer different parity for start/end values.
net.ipv4.ip_local_port_range = 32768 32768
# ./seteuid 
euid: 1001	bound to 10.0.2.15:32768
euid: 1002	bound to 10.0.2.15:32768
euid: 1001	listen succeeded
euid: 1002	listen failed
============