[PATCH 0/4] Match TCP connection timeouts to the lease period

[PATCH 0/4] Match TCP connection timeouts to the lease period

[Trond Myklebust]

With the current default TCP connection timeout being set at around
3 minutes, and most server vendors setting the lease period at
values significantly lower than that, we can end up losing the lease
while waiting for the TCP layer to discover that we need to break the
connection.
This patch series sets up an interface to allow the NFSv4 client to
adjust thsee timeout values down once it has obtained a value for
the lease period from the server.

Trond Myklebust (4):
  SUNRPC: Remove unused function rpc_get_timeout()
  SUNRPC: Refactor TCP socket timeout code into a helper function
  SUNRPC: Allow changing of the TCP timeout parameters on the fly
  NFSv4: Set the connection timeout to match the lease period

 fs/nfs/nfs4renewd.c             |  2 +-
 include/linux/sunrpc/clnt.h     |  6 +--
 include/linux/sunrpc/xprt.h     |  4 ++
 include/linux/sunrpc/xprtsock.h |  3 ++
 net/sunrpc/clnt.c               | 51 +++++++++++++-----------
 net/sunrpc/xprtsock.c           | 88 ++++++++++++++++++++++++++++++++---------
 6 files changed, 107 insertions(+), 47 deletions(-)

-- 
2.9.3

[PATCH 1/4] SUNRPC: Remove unused function rpc_get_timeout()

[Trond Myklebust]

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 include/linux/sunrpc/clnt.h |  1 -
 net/sunrpc/clnt.c           | 15 ---------------
 2 files changed, 16 deletions(-)

diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index 333ad11b3dd9..33f216edb434 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -182,7 +182,6 @@ int		rpc_protocol(struct rpc_clnt *);
 struct net *	rpc_net_ns(struct rpc_clnt *);
 size_t		rpc_max_payload(struct rpc_clnt *);
 size_t		rpc_max_bc_payload(struct rpc_clnt *);
-unsigned long	rpc_get_timeout(struct rpc_clnt *clnt);
 void		rpc_force_rebind(struct rpc_clnt *);
 size_t		rpc_peeraddr(struct rpc_clnt *, struct sockaddr *, size_t);
 const char	*rpc_peeraddr2str(struct rpc_clnt *, enum rpc_display_format_t);
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 1dc9f3bac099..2838a1fab460 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1453,21 +1453,6 @@ size_t rpc_max_bc_payload(struct rpc_clnt *clnt)
 EXPORT_SYMBOL_GPL(rpc_max_bc_payload);
 
 /**
- * rpc_get_timeout - Get timeout for transport in units of HZ
- * @clnt: RPC client to query
- */
-unsigned long rpc_get_timeout(struct rpc_clnt *clnt)
-{
-	unsigned long ret;
-
-	rcu_read_lock();
-	ret = rcu_dereference(clnt->cl_xprt)->timeout->to_initval;
-	rcu_read_unlock();
-	return ret;
-}
-EXPORT_SYMBOL_GPL(rpc_get_timeout);
-
-/**
  * rpc_force_rebind - force transport to check that remote port is unchanged
  * @clnt: client to rebind
  *
-- 
2.9.3

[PATCH 2/4] SUNRPC: Refactor TCP socket timeout code into a helper function

[Trond Myklebust]

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 net/sunrpc/xprtsock.c | 45 ++++++++++++++++++++++++++-------------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index af392d9b9cec..c8ac649a51cb 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -2235,6 +2235,31 @@ static void xs_tcp_shutdown(struct rpc_xprt *xprt)
 		xs_reset_transport(transport);
 }
 
+static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
+		struct socket *sock)
+{
+	unsigned int keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
+	unsigned int keepcnt = xprt->timeout->to_retries + 1;
+	unsigned int opt_on = 1;
+	unsigned int timeo;
+
+	/* TCP Keepalive options */
+	kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
+			(char *)&opt_on, sizeof(opt_on));
+	kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE,
+			(char *)&keepidle, sizeof(keepidle));
+	kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL,
+			(char *)&keepidle, sizeof(keepidle));
+	kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT,
+			(char *)&keepcnt, sizeof(keepcnt));
+
+	/* TCP user timeout (see RFC5482) */
+	timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
+		(xprt->timeout->to_retries + 1);
+	kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT,
+			(char *)&timeo, sizeof(timeo));
+}
+
 static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
 {
 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
@@ -2242,22 +2267,8 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
 
 	if (!transport->inet) {
 		struct sock *sk = sock->sk;
-		unsigned int keepidle = xprt->timeout->to_initval / HZ;
-		unsigned int keepcnt = xprt->timeout->to_retries + 1;
-		unsigned int opt_on = 1;
-		unsigned int timeo;
 		unsigned int addr_pref = IPV6_PREFER_SRC_PUBLIC;
 
-		/* TCP Keepalive options */
-		kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
-				(char *)&opt_on, sizeof(opt_on));
-		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE,
-				(char *)&keepidle, sizeof(keepidle));
-		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL,
-				(char *)&keepidle, sizeof(keepidle));
-		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT,
-				(char *)&keepcnt, sizeof(keepcnt));
-
 		/* Avoid temporary address, they are bad for long-lived
 		 * connections such as NFS mounts.
 		 * RFC4941, section 3.6 suggests that:
@@ -2268,11 +2279,7 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
 		kernel_setsockopt(sock, SOL_IPV6, IPV6_ADDR_PREFERENCES,
 				(char *)&addr_pref, sizeof(addr_pref));
 
-		/* TCP user timeout (see RFC5482) */
-		timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
-			(xprt->timeout->to_retries + 1);
-		kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT,
-				(char *)&timeo, sizeof(timeo));
+		xs_tcp_set_socket_timeouts(xprt, sock);
 
 		write_lock_bh(&sk->sk_callback_lock);
 
-- 
2.9.3

[PATCH 3/4] SUNRPC: Allow changing of the TCP timeout parameters on the fly

[Trond Myklebust]

When the NFSv4 server tells us the lease period, we usually want
to adjust down the timeout parameters on the TCP connection to
ensure that we don't miss lease renewals due to a faulty connection.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 include/linux/sunrpc/xprt.h     |  4 ++++
 include/linux/sunrpc/xprtsock.h |  3 +++
 net/sunrpc/clnt.c               | 30 ++++++++++++++++++------
 net/sunrpc/xprtsock.c           | 51 +++++++++++++++++++++++++++++++++++++----
 4 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index a5da60b24d83..eab1c749e192 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -137,6 +137,9 @@ struct rpc_xprt_ops {
 	void		(*release_request)(struct rpc_task *task);
 	void		(*close)(struct rpc_xprt *xprt);
 	void		(*destroy)(struct rpc_xprt *xprt);
+	void		(*set_connect_timeout)(struct rpc_xprt *xprt,
+					unsigned long connect_timeout,
+					unsigned long reconnect_timeout);
 	void		(*print_stats)(struct rpc_xprt *xprt, struct seq_file *seq);
 	int		(*enable_swap)(struct rpc_xprt *xprt);
 	void		(*disable_swap)(struct rpc_xprt *xprt);
@@ -221,6 +224,7 @@ struct rpc_xprt {
 	struct timer_list	timer;
 	unsigned long		last_used,
 				idle_timeout,
+				connect_timeout,
 				max_reconnect_timeout;
 
 	/*
diff --git a/include/linux/sunrpc/xprtsock.h b/include/linux/sunrpc/xprtsock.h
index bef3fb0abb8f..c9959d7e3579 100644
--- a/include/linux/sunrpc/xprtsock.h
+++ b/include/linux/sunrpc/xprtsock.h
@@ -55,6 +55,8 @@ struct sock_xprt {
 	size_t			rcvsize,
 				sndsize;
 
+	struct rpc_timeout	tcp_timeout;
+
 	/*
 	 * Saved socket callback addresses
 	 */
@@ -81,6 +83,7 @@ struct sock_xprt {
 
 #define XPRT_SOCK_CONNECTING	1U
 #define XPRT_SOCK_DATA_READY	(2)
+#define XPRT_SOCK_UPD_TIMEOUT	(3)
 
 #endif /* __KERNEL__ */
 
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 2838a1fab460..b5bc0c589f6a 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -2684,6 +2684,7 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt,
 {
 	struct rpc_xprt_switch *xps;
 	struct rpc_xprt *xprt;
+	unsigned long connect_timeout;
 	unsigned long reconnect_timeout;
 	unsigned char resvport;
 	int ret = 0;
@@ -2696,6 +2697,7 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt,
 		return -EAGAIN;
 	}
 	resvport = xprt->resvport;
+	connect_timeout = xprt->connect_timeout;
 	reconnect_timeout = xprt->max_reconnect_timeout;
 	rcu_read_unlock();
 
@@ -2705,7 +2707,10 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt,
 		goto out_put_switch;
 	}
 	xprt->resvport = resvport;
-	xprt->max_reconnect_timeout = reconnect_timeout;
+	if (xprt->ops->set_connect_timeout != NULL)
+		xprt->ops->set_connect_timeout(xprt,
+				connect_timeout,
+				reconnect_timeout);
 
 	rpc_xprt_switch_set_roundrobin(xps);
 	if (setup) {
@@ -2722,24 +2727,35 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt,
 }
 EXPORT_SYMBOL_GPL(rpc_clnt_add_xprt);
 
+struct connect_timeout_data {
+	unsigned long connect_timeout;
+	unsigned long reconnect_timeout;
+};
+
 static int
-rpc_xprt_cap_max_reconnect_timeout(struct rpc_clnt *clnt,
+rpc_xprt_set_connect_timeout(struct rpc_clnt *clnt,
 		struct rpc_xprt *xprt,
 		void *data)
 {
-	unsigned long timeout = *((unsigned long *)data);
+	struct connect_timeout_data *timeo = data;
 
-	if (timeout < xprt->max_reconnect_timeout)
-		xprt->max_reconnect_timeout = timeout;
+	if (xprt->ops->set_connect_timeout)
+		xprt->ops->set_connect_timeout(xprt,
+				timeo->connect_timeout,
+				timeo->reconnect_timeout);
 	return 0;
 }
 
 void
 rpc_cap_max_reconnect_timeout(struct rpc_clnt *clnt, unsigned long timeo)
 {
+	struct connect_timeout_data timeout = {
+		.connect_timeout = timeo,
+		.reconnect_timeout = timeo,
+	};
 	rpc_clnt_iterate_for_each_xprt(clnt,
-			rpc_xprt_cap_max_reconnect_timeout,
-			&timeo);
+			rpc_xprt_set_connect_timeout,
+			&timeout);
 }
 EXPORT_SYMBOL_GPL(rpc_cap_max_reconnect_timeout);
 
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index c8ac649a51cb..810e9b59be16 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -52,6 +52,8 @@
 #include "sunrpc.h"
 
 static void xs_close(struct rpc_xprt *xprt);
+static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
+		struct socket *sock);
 
 /*
  * xprtsock tunables
@@ -666,6 +668,9 @@ static int xs_tcp_send_request(struct rpc_task *task)
 	if (task->tk_flags & RPC_TASK_SENT)
 		zerocopy = false;
 
+	if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state))
+		xs_tcp_set_socket_timeouts(xprt, transport->sock);
+
 	/* Continue transmitting the packet/record. We must be careful
 	 * to cope with writespace callbacks arriving _after_ we have
 	 * called sendmsg(). */
@@ -2238,11 +2243,20 @@ static void xs_tcp_shutdown(struct rpc_xprt *xprt)
 static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
 		struct socket *sock)
 {
-	unsigned int keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
-	unsigned int keepcnt = xprt->timeout->to_retries + 1;
+	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
+	unsigned int keepidle;
+	unsigned int keepcnt;
 	unsigned int opt_on = 1;
 	unsigned int timeo;
 
+	spin_lock_bh(&xprt->transport_lock);
+	keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
+	keepcnt = xprt->timeout->to_retries + 1;
+	timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
+		(xprt->timeout->to_retries + 1);
+	clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
+	spin_unlock_bh(&xprt->transport_lock);
+
 	/* TCP Keepalive options */
 	kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
 			(char *)&opt_on, sizeof(opt_on));
@@ -2254,12 +2268,38 @@ static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
 			(char *)&keepcnt, sizeof(keepcnt));
 
 	/* TCP user timeout (see RFC5482) */
-	timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
-		(xprt->timeout->to_retries + 1);
 	kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT,
 			(char *)&timeo, sizeof(timeo));
 }
 
+static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt,
+		unsigned long connect_timeout,
+		unsigned long reconnect_timeout)
+{
+	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
+	struct rpc_timeout to;
+	unsigned long initval;
+
+	spin_lock_bh(&xprt->transport_lock);
+	if (reconnect_timeout < xprt->max_reconnect_timeout)
+		xprt->max_reconnect_timeout = reconnect_timeout;
+	if (connect_timeout < xprt->connect_timeout) {
+		memcpy(&to, xprt->timeout, sizeof(to));
+		initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1);
+		/* Arbitrary lower limit */
+		if (initval <  XS_TCP_INIT_REEST_TO << 1)
+			initval = XS_TCP_INIT_REEST_TO << 1;
+		to.to_initval = initval;
+		to.to_maxval = initval;
+		memcpy(&transport->tcp_timeout, &to,
+				sizeof(transport->tcp_timeout));
+		xprt->timeout = &transport->tcp_timeout;
+		xprt->connect_timeout = connect_timeout;
+	}
+	set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
+	spin_unlock_bh(&xprt->transport_lock);
+}
+
 static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
 {
 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
@@ -2728,6 +2768,7 @@ static struct rpc_xprt_ops xs_tcp_ops = {
 	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
 	.close			= xs_tcp_shutdown,
 	.destroy		= xs_destroy,
+	.set_connect_timeout	= xs_tcp_set_connect_timeout,
 	.print_stats		= xs_tcp_print_stats,
 	.enable_swap		= xs_enable_swap,
 	.disable_swap		= xs_disable_swap,
@@ -3014,6 +3055,8 @@ static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args)
 	xprt->timeout = &xs_tcp_default_timeout;
 
 	xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
+	xprt->connect_timeout = xprt->timeout->to_initval *
+		(xprt->timeout->to_retries + 1);
 
 	INIT_WORK(&transport->recv_worker, xs_tcp_data_receive_workfn);
 	INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket);
-- 
2.9.3

[PATCH 4/4] NFSv4: Set the connection timeout to match the lease period

[Trond Myklebust]

Set the timeout for TCP connections to be 1 lease period to ensure
that we don't lose our lease due to a faulty TCP connection.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 fs/nfs/nfs4renewd.c         |  2 +-
 include/linux/sunrpc/clnt.h |  5 +++--
 net/sunrpc/clnt.c           | 10 ++++++----
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/fs/nfs/nfs4renewd.c b/fs/nfs/nfs4renewd.c
index 82e77198d17e..1f8c2ae43a8d 100644
--- a/fs/nfs/nfs4renewd.c
+++ b/fs/nfs/nfs4renewd.c
@@ -153,7 +153,7 @@ void nfs4_set_lease_period(struct nfs_client *clp,
 	spin_unlock(&clp->cl_lock);
 
 	/* Cap maximum reconnect timeout at 1/2 lease period */
-	rpc_cap_max_reconnect_timeout(clp->cl_rpcclient, lease >> 1);
+	rpc_set_connect_timeout(clp->cl_rpcclient, lease, lease >> 1);
 }
 
 /*
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index 33f216edb434..6095ecba0dde 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -201,8 +201,9 @@ int		rpc_clnt_add_xprt(struct rpc_clnt *, struct xprt_create *,
 				struct rpc_xprt *,
 				void *),
 			void *data);
-void		rpc_cap_max_reconnect_timeout(struct rpc_clnt *clnt,
-			unsigned long timeo);
+void		rpc_set_connect_timeout(struct rpc_clnt *clnt,
+			unsigned long connect_timeout,
+			unsigned long reconnect_timeout);
 
 int		rpc_clnt_setup_test_and_add_xprt(struct rpc_clnt *,
 			struct rpc_xprt_switch *,
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index b5bc0c589f6a..52da3ce54bb5 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -2747,17 +2747,19 @@ rpc_xprt_set_connect_timeout(struct rpc_clnt *clnt,
 }
 
 void
-rpc_cap_max_reconnect_timeout(struct rpc_clnt *clnt, unsigned long timeo)
+rpc_set_connect_timeout(struct rpc_clnt *clnt,
+		unsigned long connect_timeout,
+		unsigned long reconnect_timeout)
 {
 	struct connect_timeout_data timeout = {
-		.connect_timeout = timeo,
-		.reconnect_timeout = timeo,
+		.connect_timeout = connect_timeout,
+		.reconnect_timeout = reconnect_timeout,
 	};
 	rpc_clnt_iterate_for_each_xprt(clnt,
 			rpc_xprt_set_connect_timeout,
 			&timeout);
 }
-EXPORT_SYMBOL_GPL(rpc_cap_max_reconnect_timeout);
+EXPORT_SYMBOL_GPL(rpc_set_connect_timeout);
 
 void rpc_clnt_xprt_switch_put(struct rpc_clnt *clnt)
 {
-- 
2.9.3

Re: [PATCH 0/4] Match TCP connection timeouts to the lease period

[Trond Myklebust]

T24gVGh1LCAyMDE3LTAyLTA5IGF0IDA4OjQ3ICswMTAwLCBNa3J0Y2h5YW4sIFRpZ3JhbiB3cm90
ZToNCj4gDQo+IC0tLS0tIE9yaWdpbmFsIE1lc3NhZ2UgLS0tLS0NCj4gPiBGcm9tOiAiVHJvbmQg
TXlrbGVidXN0IiA8dHJvbmRteUBwcmltYXJ5ZGF0YS5jb20+DQo+ID4gVG86ICJ0aWdyYW4gbWty
dGNoeWFuIiA8dGlncmFuLm1rcnRjaHlhbkBkZXN5LmRlPg0KPiA+IENjOiAiQW5uYSBTY2h1bWFr
ZXIiIDxBbm5hLlNjaHVtYWtlckBuZXRhcHAuY29tPiwgbGludXgtbmZzQHZnZXIua2UNCj4gPiBy
bmVsLm9yZw0KPiA+IFNlbnQ6IFdlZG5lc2RheSwgRmVicnVhcnkgOCwgMjAxNyAxMDo1Nzo0NCBQ
TQ0KPiA+IFN1YmplY3Q6IFJlOiBbUEFUQ0ggMC80XSBNYXRjaCBUQ1AgY29ubmVjdGlvbiB0aW1l
b3V0cyB0byB0aGUgbGVhc2UNCj4gPiBwZXJpb2QNCj4gPiBPbiBXZWQsIDIwMTctMDItMDggYXQg
MjI6MTggKzAxMDAsIE1rcnRjaHlhbiwgVGlncmFuIHdyb3RlOg0KPiA+ID4gDQo+ID4gPiANCj4g
PiA+IE9uIEZlYiA4LCAyMDE3IDE3OjQ4LCBUcm9uZCBNeWtsZWJ1c3QgPHRyb25kLm15a2xlYnVz
dEBwcmltYXJ5ZGF0DQo+ID4gPiBhLmNvDQo+ID4gPiBtPiB3cm90ZToNCj4gPiA+IFdpdGggdGhl
IGN1cnJlbnQgZGVmYXVsdCBUQ1AgY29ubmVjdGlvbiB0aW1lb3V0IGJlaW5nIHNldCBhdA0KPiA+
ID4gYXJvdW5kDQo+ID4gPiAzIG1pbnV0ZXMsIGFuZCBtb3N0IHNlcnZlciB2ZW5kb3JzIHNldHRp
bmcgdGhlIGxlYXNlIHBlcmlvZCBhdA0KPiA+ID4gdmFsdWVzIHNpZ25pZmljYW50bHkgbG93ZXIg
dGhhbiB0aGF0LCB3ZSBjYW4gZW5kIHVwIGxvc2luZyB0aGUNCj4gPiA+IGxlYXNlDQo+ID4gPiB3
aGlsZSB3YWl0aW5nIGZvciB0aGUgVENQIGxheWVyIHRvIGRpc2NvdmVyIHRoYXQgd2UgbmVlZCB0
byBicmVhaw0KPiA+ID4gdGhlDQo+ID4gPiBjb25uZWN0aW9uLg0KPiA+ID4gVGhpcyBwYXRjaCBz
ZXJpZXMgc2V0cyB1cCBhbiBpbnRlcmZhY2UgdG8gYWxsb3cgdGhlIE5GU3Y0IGNsaWVudA0KPiA+
ID4gdG8NCj4gPiA+IGFkanVzdCB0aHNlZSB0aW1lb3V0IHZhbHVlcyBkb3duIG9uY2UgaXQgaGFz
IG9idGFpbmVkIGEgdmFsdWUgZm9yDQo+ID4gPiB0aGUgbGVhc2UgcGVyaW9kIGZyb20gdGhlIHNl
cnZlci4NCj4gPiA+IA0KPiA+ID4gDQo+ID4gPiBDb29sIGZpbmFsbHkhIEkgd2FzIHdhaXRpbmcg
Zm9yIHRoYXQgcXVpdGUgYSBiaXQuDQo+ID4gPiANCj4gPiA+IGh0dHBzOi8vd3d3LmlldGYub3Jn
L21haWwtYXJjaGl2ZS93ZWIvbmZzdjQvY3VycmVudC9tc2cxMzc1OC5odG1sDQo+ID4gPiANCj4g
PiA+IA0KPiA+IA0KPiA+IE5vLiBUaGlzIG1lY2hhbmlzbSBkb2VzIG5vdCBleGN1c2UgdGhlIHNl
cnZlciBmcm9tIGhhdmluZyB0bw0KPiA+IHByb2Nlc3MNCj4gPiByZXF1ZXN0cywgYW5kIEkgYW0g
Y2VydGFpbmx5IG5vdCBzaWduaW5nIHVzIHVwIHRvIGFueSAiZ2VudGxlbWFuJ3MNCj4gPiBhZ3Jl
ZW1lbnQiLg0KPiA+IA0KPiA+IElmIHRoZSBzZXJ2ZXIgaGFzIHJlY2VpdmVkIGFuIFJQQyBjYWxs
LCB0aGVuIGl0IE1VU1QgcHJvY2VzcyBpdCwNCj4gPiB3aGV0aGVyIG9yIG5vdCB0aGVyZSBpcyBh
IFRDUCBjb25uZWN0aW9uLCBvdGhlcndpc2UgdGhlIGxlYXNlIG1heQ0KPiA+IHN0aWxsDQo+ID4g
YmUgbG9zdCBkdWUgdG8gdGhlIHNlcnZlciBmYWlsaW5nIHRvIGxpdmUgdXAgdG8gdGhlIHJlcXVp
cmVtZW50cyBpbg0KPiA+IFJGQzU2NjEgU2VjdGlvbiA4LjPCoChTZWUgaHR0cHM6Ly90b29scy5p
ZXRmLm9yZy9odG1sL3JmYzU2NjEjc2VjdGlvDQo+ID4gbi04Lg0KPiA+IDMpLiBUaG9zZSByZXF1
aXJlbWVudHMgc3RhdGUgdGhhdCAiSWYgdGhlIGNsaWVudCBJRCdzIGxlYXNlIGhhcyBub3QNCj4g
PiBleHBpcmVkIHdoZW4gdGhlIHNlcnZlciByZWNlaXZlcyBhIFNFUVVFTkNFIG9wZXJhdGlvbiwg
dGhlbiB0aGUNCj4gPiBzZXJ2ZXINCj4gPiBNVVNUIHJlbmV3IHRoZSBsZWFzZS4iDQo+ID4gDQo+
ID4gVGhlIHBvaW50IG9mIHRoZXNlIHBhdGNoZXMgYXJlIHRvIGVuc3VyZSB0aGF0IHdlIGRldGVj
dCByb3V0aW5nDQo+ID4gY2hhbmdlcw0KPiA+IGFuZCB0aGluZ3MgbGlrZSB0aGF0IGluIGEgdGlt
ZWx5IGZhc2hpb24gKGFnYWluLCBhcyByZXF1aXJlZCBieQ0KPiA+IFJGQzU2NjEsIFNlY3Rpb24g
OC4zKSBzbyB0aGF0IHdlIGNhbiByZS1lc3RhYmxpc2ggdGhlIGNvbm5lY3Rpb24NCj4gPiBhbmQN
Cj4gPiByZW5ldyB0aGUgbGVhc2UuDQo+ID4gDQo+IA0KPiBTdXJlLCBJIGFtIGZ1bGx5IGFncmVl
IHdpdGggeW91LiBNeSBwb2ludCBpcyB0aGF0IGNsaWVudCdzIFRDUA0KPiB0aW1lb3V0DQo+IGlz
IG5vdCBhICdyYW5kb20gbnVtYmVyJyBhbnkgbW9yZSBhbmQgaGF2ZSBhIHJlbGF0aW9uIHdpdGgg
dGhlIGxlYXNlDQo+IHRpbWUuDQo+IEhvd2V2ZXIsIHRoZSBzYW1lIGNhbiBiZSBhcHBsaWVkIHRv
IFJQQyB0aW1lb3V0cyBhcyB3ZWxsLiBPdXIgc2VydmVyDQo+IGFsd2F5cyB0byBzZW5kcyBFUlJf
REVMQVkgaWYgcmVxdWVzdCBpcyBub3QgcHJvY2Vzc2VkIHdpdGhpbiAzc2VjIHRvDQo+IGF2b2lk
IHJldHJpZXMuIFRoaXMgJzNzZWMnIGlzIGEgcmFuZG9tIG51bWJlciwgYXMgd2UgaGF2ZSBubyBp
ZGVhDQo+IHdoZW4NCj4gdGhlIGNsaWVudCB3aWxsIGxvc2UgaGlzIHBhdGllbmNlLg0KPiANCg0K
RmFpciBlbm91Z2guIE5vdGUgdGhhdCB0aGUgY2xpZW50IGlzIHJlcXVpcmVkIHRvIHdhaXQgdW50
aWwgdGhlIFJQQw0KY2FsbCBpcyBmaW5pc2hlZCBubyBtYXR0ZXIgd2hhdDsgaXQgY2Fubm90IHJl
dXNlIHRoYXQgc2xvdCB1bnRpbCB0aGUNCnNlcnZlciBpcyBkb25lLg0KDQpBcyBmb3IgdGhlIFRD
UCBjb25uZWN0aW9uIHRpbWVvdXQsIGl0IGlzIHN0aWxsIGEgbGl0dGxlIHVuY2xlYXIgdG8gbWUN
CnRoYXQgdGhlIDEgbGVhc2UgcGVyaW9kIGlzIG9wdGltYWwuIFlvdSBjYW4gc3RpbGwgZW5kIHVw
IGxvc2luZyB0aGUNCmxlYXNlLCBzaW5jZSB3ZSBkb24ndCBzdGFydCBzZW5kaW5nIFNFUVVFTkNF
IG9wZXJhdGlvbnMgdW50aWwgd2VsbCBpbnRvDQp0aGUgc2Vjb25kIGhhbGYgb2YgdGhlIHBlcmlv
ZCBmb2xsb3dpbmcgdGhlIGxhc3QgbGVhc2UgcmVuZXdhbC4gTXkgaG9wZQ0KaXMgdGhhdCB0aGUg
S0VFUEFMSVZFIHdpbGwgc2F2ZSB1cyB0aGVyZSwgc2luY2UgaXQgbW9uaXRvcnMgdGhlIGVudGly
ZQ0KbGVhc2UgcGVyaW9kLg0KVGhhdCBzYWlkLCBJJ20gb3BlbiB0byBhcmd1bWVudHMgdGhhdCB3
ZSBtaWdodCB3YW50IHRvIHJlZHVjZSB0aGUNCnRpbWVvdXQgZnVydGhlciB0bywgc2F5LCAzLzQg
bGVhc2UgcGVyaW9kIGluIG9yZGVyIHRvIGFsbG93IGZvciBzb21lDQp0aW1lIHRvIHJlLWVzdGFi
bGlzaCB0aGUgVENQIGNvbm5lY3Rpb24uIFRoZSByZWFzb24gd2h5IEkgaGF2ZSBub3QgZG9uZQ0K
dGhpcyBpcyBiZWNhdXNlIHRoYXQgbWVhbnMgd2Ugd2lsbCBlbmQgdXAgYnJlYWtpbmcgdGhlIGNv
bm5lY3Rpb24gYWZ0ZXINCjMwcyBhZ2FpbnN0IHNvbWUgc2VydmVyczsgSSdtIG5vdCBzdXJlIHRo
YXQgZXZlbiBkYXRhIGNlbnRyZSBuZXR3b3Jrcw0Kd2lsbCBndWFyYW50ZWUgc3RhYmlsaXR5IG9m
IHNlcnZpY2UgYXQgdGhhdCBsZXZlbC4NCg0KVGhvdWdodHMgYW5kIGNvbW1lbnRzPw0KDQo+IFRp
Z3Jhbi4NCj4gDQo+IA0KPiA+ID4gVGlncmFuLg0KPiA+ID4gDQo+ID4gPiBUcm9uZCBNeWtsZWJ1
c3QgKDQpOg0KPiA+ID4gwqAgU1VOUlBDOiBSZW1vdmUgdW51c2VkIGZ1bmN0aW9uIHJwY19nZXRf
dGltZW91dCgpDQo+ID4gPiDCoCBTVU5SUEM6IFJlZmFjdG9yIFRDUCBzb2NrZXQgdGltZW91dCBj
b2RlIGludG8gYSBoZWxwZXIgZnVuY3Rpb24NCj4gPiA+IMKgIFNVTlJQQzogQWxsb3cgY2hhbmdp
bmcgb2YgdGhlIFRDUCB0aW1lb3V0IHBhcmFtZXRlcnMgb24gdGhlIGZseQ0KPiA+ID4gwqAgTkZT
djQ6IFNldCB0aGUgY29ubmVjdGlvbiB0aW1lb3V0IHRvIG1hdGNoIHRoZSBsZWFzZSBwZXJpb2QN
Cj4gPiA+IA0KPiA+ID4gZnMvbmZzL25mczRyZW5ld2QuY8KgwqDCoMKgwqDCoMKgwqDCoMKgwqDC
oCB8wqAgMiArLQ0KPiA+ID4gaW5jbHVkZS9saW51eC9zdW5ycGMvY2xudC5owqDCoMKgwqAgfMKg
IDYgKy0tDQo+ID4gPiBpbmNsdWRlL2xpbnV4L3N1bnJwYy94cHJ0LmjCoMKgwqDCoCB8wqAgNCAr
Kw0KPiA+ID4gaW5jbHVkZS9saW51eC9zdW5ycGMveHBydHNvY2suaCB8wqAgMyArKw0KPiA+ID4g
bmV0L3N1bnJwYy9jbG50LmPCoMKgwqDCoMKgwqDCoMKgwqDCoMKgwqDCoMKgIHwgNTEgKysrKysr
KysrKysrKy0tLS0tLS0tLS0tDQo+ID4gPiBuZXQvc3VucnBjL3hwcnRzb2NrLmPCoMKgwqDCoMKg
wqDCoMKgwqDCoCB8IDg4DQo+ID4gPiArKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKy0t
LS0tLS0tLQ0KPiA+ID4gNiBmaWxlcyBjaGFuZ2VkLCAxMDcgaW5zZXJ0aW9ucygrKSwgNDcgZGVs
ZXRpb25zKC0pDQo+ID4gPiANCj4gPiANCj4gPiAtLQ0KPiA+IA0KPiA+IA0KPiA+IAkNCj4gPiAJ
DQo+ID4gDQo+ID4gDQo+ID4gVHJvbmQgTXlrbGVidXN0DQo+ID4gUHJpbmNpcGFsIFN5c3RlbSBB
cmNoaXRlY3QNCj4gPiA0MzAwIEVsIENhbWlubyBSZWFsIHwgU3VpdGUgMTAwDQo+ID4gTG9zIEFs
dG9zLCBDQcKgwqA5NDAyMg0KPiA+IFc6IDY1MC00MjItMzgwMA0KPiA+IEM6IDgwMS05MjEtNDU4
Mw0KPiA+IHd3dy5wcmltYXJ5ZGF0YS5jb20NCj4gPiANCj4gPiANCj4gPiANCj4gPiBO77+977+9
77+977+977+9cu+/ve+/vXnvv73vv73vv71i77+9WO+/ve+/vcendu+/vV7vv70p3rp7Lm7vv70r
77+977+977+977+9e++/ve+/ve+/vSLvv73vv71ebu+/vXLvv73vv73vv71677+977+977+9aO+/
ve+/ve+/ve+/vSbvv73vv73vv71H77+977+977+9aO+/vQ0KPiA+ICjvv73pmo7vv73domoi77+9
77+977+9be+/ve+/ve+/ve+/ve+/vXrvv73elu+/ve+/ve+/vWbvv73vv73vv71o77+977+977+9
fu+/vW3vv70NCj4gDQo+IA0KLS0gDQoNCg0KDQoJDQoJDQoNCg0KVHJvbmQgTXlrbGVidXN0DQpQ
cmluY2lwYWwgU3lzdGVtIEFyY2hpdGVjdA0KNDMwMCBFbCBDYW1pbm8gUmVhbCB8IFN1aXRlIDEw
MA0KTG9zIEFsdG9zLCBDQcKgwqA5NDAyMg0KVzogNjUwLTQyMi0zODAwDQpDOiA4MDEtOTIxLTQ1
ODPCoA0Kd3d3LnByaW1hcnlkYXRhLmNvbQ0KDQoNCg0KDQo=

Re: [PATCH 0/4] Match TCP connection timeouts to the lease period

[Mkrtchyan, Tigran]

----- Original Message -----
> From: "Trond Myklebust" <trondmy@primarydata.com>
> To: "tigran mkrtchyan" <tigran.mkrtchyan@desy.de>
> Cc: "Anna Schumaker" <Anna.Schumaker@netapp.com>, linux-nfs@vger.kernel.o=
rg
> Sent: Wednesday, February 8, 2017 10:57:44 PM
> Subject: Re: [PATCH 0/4] Match TCP connection timeouts to the lease perio=
d

> On Wed, 2017-02-08 at 22:18 +0100, Mkrtchyan, Tigran wrote:
>>=20
>>=20
>> On Feb 8, 2017 17:48, Trond Myklebust <trond.myklebust@primarydata.co
>> m> wrote:
>> With the current default TCP connection timeout being set at around
>> 3 minutes, and most server vendors setting the lease period at
>> values significantly lower than that, we can end up losing the lease
>> while waiting for the TCP layer to discover that we need to break
>> the
>> connection.
>> This patch series sets up an interface to allow the NFSv4 client to
>> adjust thsee timeout values down once it has obtained a value for
>> the lease period from the server.
>>=20
>>=20
>> Cool finally! I was waiting for that quite a bit.
>>=20
>> https://www.ietf.org/mail-archive/web/nfsv4/current/msg13758.html
>>=20
>>=20
>=20
> No. This mechanism does not excuse the server from having to process
> requests, and I am certainly not signing us up to any "gentleman's
> agreement".
>=20
> If the server has received an RPC call, then it MUST process it,
> whether or not there is a TCP connection, otherwise the lease may still
> be lost due to the server failing to live up to the requirements in
> RFC5661 Section 8.3=C2=A0(See https://tools.ietf.org/html/rfc5661#section=
-8.
> 3). Those requirements state that "If the client ID's lease has not
> expired when the server receives a SEQUENCE operation, then the server
> MUST renew the lease."
>=20
> The point of these patches are to ensure that we detect routing changes
> and things like that in a timely fashion (again, as required by
> RFC5661, Section 8.3) so that we can re-establish the connection and
> renew the lease.
>=20

Sure, I am fully agree with you. My point is that client's TCP timeout
is not a 'random number' any more and have a relation with the lease time.
However, the same can be applied to RPC timeouts as well. Our server
always to sends ERR_DELAY if request is not processed within 3sec to
avoid retries. This '3sec' is a random number, as we have no idea when
the client will lose his patience.

Tigran.


>> Tigran.
>>=20
>> Trond Myklebust (4):
>> =C2=A0 SUNRPC: Remove unused function rpc_get_timeout()
>> =C2=A0 SUNRPC: Refactor TCP socket timeout code into a helper function
>> =C2=A0 SUNRPC: Allow changing of the TCP timeout parameters on the fly
>> =C2=A0 NFSv4: Set the connection timeout to match the lease period
>>=20
>> fs/nfs/nfs4renewd.c=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=
=A0=C2=A0=C2=A0=C2=A0 |=C2=A0 2 +-
>> include/linux/sunrpc/clnt.h=C2=A0=C2=A0=C2=A0=C2=A0 |=C2=A0 6 +--
>> include/linux/sunrpc/xprt.h=C2=A0=C2=A0=C2=A0=C2=A0 |=C2=A0 4 ++
>> include/linux/sunrpc/xprtsock.h |=C2=A0 3 ++
>> net/sunrpc/clnt.c=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=
=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0 | 51 +++++++++++++-----------
>> net/sunrpc/xprtsock.c=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=
=A0=C2=A0 | 88
>> ++++++++++++++++++++++++++++++++---------
>> 6 files changed, 107 insertions(+), 47 deletions(-)
>>=20
> --
>=20
>=20
>=09
>=09
>=20
>=20
> Trond Myklebust
> Principal System Architect
> 4300 El Camino Real | Suite 100
> Los Altos, CA=C2=A0=C2=A094022
> W: 650-422-3800
> C: 801-921-4583
> www.primarydata.com
>=20
>=20
>=20
> N=EF=BF=BD=EF=BF=BD=EF=BF=BD=EF=BF=BD=EF=BF=BDr=EF=BF=BD=EF=BF=BDy=EF=BF=
=BD=EF=BF=BD=EF=BF=BDb=EF=BF=BDX=EF=BF=BD=EF=BF=BD=C7=A7v=EF=BF=BD^=EF=BF=
=BD)=DE=BA{.n=EF=BF=BD+=EF=BF=BD=EF=BF=BD=EF=BF=BD=EF=BF=BD{=EF=BF=BD=EF=BF=
=BD=EF=BF=BD"=EF=BF=BD=EF=BF=BD^n=EF=BF=BDr=EF=BF=BD=EF=BF=BD=EF=BF=BDz=EF=
=BF=BD=EF=BF=BD=EF=BF=BDh=EF=BF=BD=EF=BF=BD=EF=BF=BD=EF=BF=BD&=EF=BF=BD=EF=
=BF=BD=EF=BF=BDG=EF=BF=BD=EF=BF=BD=EF=BF=BDh=EF=BF=BD(=EF=BF=BD=E9=9A=8E=EF=
=BF=BD=DD=A2j"=EF=BF=BD=EF=BF=BD=EF=BF=BDm=EF=BF=BD=EF=BF=BD=EF=BF=BD=EF=BF=
=BD=EF=BF=BDz=EF=BF=BD=DE=96=EF=BF=BD=EF=BF=BD=EF=BF=BDf=EF=BF=BD=EF=BF=BD=
=EF=BF=BDh=EF=BF=BD=EF=BF=BD=EF=BF=BD~=EF=BF=BDm=EF=BF=BD

Re: [PATCH 0/4] Match TCP connection timeouts to the lease period

[Trond Myklebust]

T24gV2VkLCAyMDE3LTAyLTA4IGF0IDIyOjE4ICswMTAwLCBNa3J0Y2h5YW4sIFRpZ3JhbiB3cm90
ZToNCj4gDQo+IA0KPiBPbiBGZWIgOCwgMjAxNyAxNzo0OCwgVHJvbmQgTXlrbGVidXN0IDx0cm9u
ZC5teWtsZWJ1c3RAcHJpbWFyeWRhdGEuY28NCj4gbT4gd3JvdGU6DQo+IFdpdGggdGhlIGN1cnJl
bnQgZGVmYXVsdCBUQ1AgY29ubmVjdGlvbiB0aW1lb3V0IGJlaW5nIHNldCBhdCBhcm91bmTCoA0K
PiAzIG1pbnV0ZXMsIGFuZCBtb3N0IHNlcnZlciB2ZW5kb3JzIHNldHRpbmcgdGhlIGxlYXNlIHBl
cmlvZCBhdMKgDQo+IHZhbHVlcyBzaWduaWZpY2FudGx5IGxvd2VyIHRoYW4gdGhhdCwgd2UgY2Fu
IGVuZCB1cCBsb3NpbmcgdGhlIGxlYXNlwqANCj4gd2hpbGUgd2FpdGluZyBmb3IgdGhlIFRDUCBs
YXllciB0byBkaXNjb3ZlciB0aGF0IHdlIG5lZWQgdG8gYnJlYWsNCj4gdGhlwqANCj4gY29ubmVj
dGlvbi7CoA0KPiBUaGlzIHBhdGNoIHNlcmllcyBzZXRzIHVwIGFuIGludGVyZmFjZSB0byBhbGxv
dyB0aGUgTkZTdjQgY2xpZW50IHRvwqANCj4gYWRqdXN0IHRoc2VlIHRpbWVvdXQgdmFsdWVzIGRv
d24gb25jZSBpdCBoYXMgb2J0YWluZWQgYSB2YWx1ZSBmb3LCoA0KPiB0aGUgbGVhc2UgcGVyaW9k
IGZyb20gdGhlIHNlcnZlci7CoA0KPiANCj4gDQo+IENvb2wgZmluYWxseSEgSSB3YXMgd2FpdGlu
ZyBmb3IgdGhhdCBxdWl0ZSBhIGJpdC4NCj4gDQo+IGh0dHBzOi8vd3d3LmlldGYub3JnL21haWwt
YXJjaGl2ZS93ZWIvbmZzdjQvY3VycmVudC9tc2cxMzc1OC5odG1sDQo+IA0KPiANCg0KTm8uIFRo
aXMgbWVjaGFuaXNtIGRvZXMgbm90IGV4Y3VzZSB0aGUgc2VydmVyIGZyb20gaGF2aW5nIHRvIHBy
b2Nlc3MNCnJlcXVlc3RzLCBhbmQgSSBhbSBjZXJ0YWlubHkgbm90IHNpZ25pbmcgdXMgdXAgdG8g
YW55ICJnZW50bGVtYW4ncw0KYWdyZWVtZW50Ii4NCg0KSWYgdGhlIHNlcnZlciBoYXMgcmVjZWl2
ZWQgYW4gUlBDIGNhbGwsIHRoZW4gaXQgTVVTVCBwcm9jZXNzIGl0LA0Kd2hldGhlciBvciBub3Qg
dGhlcmUgaXMgYSBUQ1AgY29ubmVjdGlvbiwgb3RoZXJ3aXNlIHRoZSBsZWFzZSBtYXkgc3RpbGwN
CmJlIGxvc3QgZHVlIHRvIHRoZSBzZXJ2ZXIgZmFpbGluZyB0byBsaXZlIHVwIHRvIHRoZSByZXF1
aXJlbWVudHMgaW4NClJGQzU2NjEgU2VjdGlvbiA4LjPCoChTZWUgaHR0cHM6Ly90b29scy5pZXRm
Lm9yZy9odG1sL3JmYzU2NjEjc2VjdGlvbi04Lg0KMykuIFRob3NlIHJlcXVpcmVtZW50cyBzdGF0
ZSB0aGF0ICJJZiB0aGUgY2xpZW50IElEJ3MgbGVhc2UgaGFzIG5vdA0KZXhwaXJlZCB3aGVuIHRo
ZSBzZXJ2ZXIgcmVjZWl2ZXMgYSBTRVFVRU5DRSBvcGVyYXRpb24sIHRoZW4gdGhlIHNlcnZlcg0K
TVVTVCByZW5ldyB0aGUgbGVhc2UuIg0KDQpUaGUgcG9pbnQgb2YgdGhlc2UgcGF0Y2hlcyBhcmUg
dG8gZW5zdXJlIHRoYXQgd2UgZGV0ZWN0IHJvdXRpbmcgY2hhbmdlcw0KYW5kIHRoaW5ncyBsaWtl
IHRoYXQgaW4gYSB0aW1lbHkgZmFzaGlvbiAoYWdhaW4sIGFzIHJlcXVpcmVkIGJ5DQpSRkM1NjYx
LCBTZWN0aW9uIDguMykgc28gdGhhdCB3ZSBjYW4gcmUtZXN0YWJsaXNoIHRoZSBjb25uZWN0aW9u
IGFuZA0KcmVuZXcgdGhlIGxlYXNlLg0KDQo+IFRpZ3Jhbi4NCj4gDQo+IFRyb25kIE15a2xlYnVz
dCAoNCk6wqANCj4gwqAgU1VOUlBDOiBSZW1vdmUgdW51c2VkIGZ1bmN0aW9uIHJwY19nZXRfdGlt
ZW91dCgpwqANCj4gwqAgU1VOUlBDOiBSZWZhY3RvciBUQ1Agc29ja2V0IHRpbWVvdXQgY29kZSBp
bnRvIGEgaGVscGVyIGZ1bmN0aW9uwqANCj4gwqAgU1VOUlBDOiBBbGxvdyBjaGFuZ2luZyBvZiB0
aGUgVENQIHRpbWVvdXQgcGFyYW1ldGVycyBvbiB0aGUgZmx5wqANCj4gwqAgTkZTdjQ6IFNldCB0
aGUgY29ubmVjdGlvbiB0aW1lb3V0IHRvIG1hdGNoIHRoZSBsZWFzZSBwZXJpb2TCoA0KPiANCj4g
ZnMvbmZzL25mczRyZW5ld2QuY8KgwqDCoMKgwqDCoMKgwqDCoMKgwqDCoCB8wqAgMiArLcKgDQo+
IGluY2x1ZGUvbGludXgvc3VucnBjL2NsbnQuaMKgwqDCoMKgIHzCoCA2ICstLcKgDQo+IGluY2x1
ZGUvbGludXgvc3VucnBjL3hwcnQuaMKgwqDCoMKgIHzCoCA0ICsrwqANCj4gaW5jbHVkZS9saW51
eC9zdW5ycGMveHBydHNvY2suaCB8wqAgMyArK8KgDQo+IG5ldC9zdW5ycGMvY2xudC5jwqDCoMKg
wqDCoMKgwqDCoMKgwqDCoMKgwqDCoCB8IDUxICsrKysrKysrKysrKystLS0tLS0tLS0tLcKgDQo+
IG5ldC9zdW5ycGMveHBydHNvY2suY8KgwqDCoMKgwqDCoMKgwqDCoMKgIHwgODgNCj4gKysrKysr
KysrKysrKysrKysrKysrKysrKysrKysrKystLS0tLS0tLS3CoA0KPiA2IGZpbGVzIGNoYW5nZWQs
IDEwNyBpbnNlcnRpb25zKCspLCA0NyBkZWxldGlvbnMoLSnCoA0KPiANCi0tIA0KDQoNCgkNCgkN
Cg0KDQpUcm9uZCBNeWtsZWJ1c3QNClByaW5jaXBhbCBTeXN0ZW0gQXJjaGl0ZWN0DQo0MzAwIEVs
IENhbWlubyBSZWFsIHwgU3VpdGUgMTAwDQpMb3MgQWx0b3MsIENBwqDCoDk0MDIyDQpXOiA2NTAt
NDIyLTM4MDANCkM6IDgwMS05MjEtNDU4M8KgDQp3d3cucHJpbWFyeWRhdGEuY29tDQoNCg0KDQo=