[PATCH v2 0/3] Do not hold clnt_fd_lock mutex during connect

[PATCH v2 0/3] Do not hold clnt_fd_lock mutex during connect

[Paulo Andrade]

  The original patch was  split in 3 new patches,  addressing some  concerns
brough in the first  version, about  thread safety of data accessed  without
the lock held.

  It was also added an extra  change to save the errno value before  calling
syslog.

  Original description of what the problem corrects follows:

  An user  reports  that  their  application  connects to  multiple  servers
through a rpc interface using  libtirpc. When one of  the servers misbehaves
(goes down  ungracefully or  has a  delay of  a few  seconds in  the traffic
flow), it was observed that the traffic from the  client to other servers is
decreased by  the  traffic  anomaly  of  the failing  server,  i.e.  traffic
decreases or goes to 0 in all the servers.

  When investigated further, specifically into the behavior  of the libtirpc
at the  time of  the issue,  it  was observed  that all  of the  application
threads specifically interacting with  libtirpc were locked into  one single
lock inside  the  libtirpc library.  This  was a  race  condition which  had
resulted in a deadlock and hence the resultant dip/stoppage of traffic.

  As an experiment, the user removed the libtirpc from the application build
and used the  standard glibc  library for rpc  communication. In  that case,
everything worked perfectly even  in the time  of the issue of  server nodes
misbehaving.

Paulo Andrade (3):
  Make it clear rpc_createerr is thread safe
  Record errno value before calling syslog
  Do not hold a global mutex during connect

 src/clnt_vc.c | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

-- 
1.8.3.1

[PATCH 1/3] Make it clear rpc_createerr is thread safe

[Paulo Andrade]

  Avoid hidding it under a macro, and also avoid multiple function
calls when accessing structure fields.

Signed-off-by: Paulo Andrade <pcpa@gnu.org>
---
 src/clnt_vc.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/clnt_vc.c b/src/clnt_vc.c
index a72f9f7..8af7ddd 100644
--- a/src/clnt_vc.c
+++ b/src/clnt_vc.c
@@ -72,6 +72,8 @@
 #define CMGROUP_MAX    16
 #define SCM_CREDS      0x03            /* process creds (struct cmsgcred) */
 
+#undef rpc_createerr                   /* make it clear it is a thread safe variable */
+
 /*
  * Credentials structure, used to verify the identity of a peer
  * process that has sent us a message. This is allocated by the
@@ -188,10 +190,11 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 	cl = (CLIENT *)mem_alloc(sizeof (*cl));
 	ct = (struct ct_data *)mem_alloc(sizeof (*ct));
 	if ((cl == (CLIENT *)NULL) || (ct == (struct ct_data *)NULL)) {
+		struct rpc_createerr *ce = &get_rpc_createerr();
 		(void) syslog(LOG_ERR, clnt_vc_errstr,
 		    clnt_vc_str, __no_mem_str);
-		rpc_createerr.cf_stat = RPC_SYSTEMERROR;
-		rpc_createerr.cf_error.re_errno = errno;
+		ce->cf_stat = RPC_SYSTEMERROR;
+		ce->cf_error.re_errno = errno;
 		goto err;
 	}
 	ct->ct_addr.buf = NULL;
@@ -235,15 +238,17 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 	slen = sizeof ss;
 	if (getpeername(fd, (struct sockaddr *)&ss, &slen) < 0) {
 		if (errno != ENOTCONN) {
-			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
-			rpc_createerr.cf_error.re_errno = errno;
+			struct rpc_createerr *ce = &get_rpc_createerr();
+			ce->cf_stat = RPC_SYSTEMERROR;
+			ce->cf_error.re_errno = errno;
 			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
 			goto err;
 		}
 		if (connect(fd, (struct sockaddr *)raddr->buf, raddr->len) < 0){
-			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
-			rpc_createerr.cf_error.re_errno = errno;
+			struct rpc_createerr *ce = &get_rpc_createerr();
+			ce->cf_stat = RPC_SYSTEMERROR;
+			ce->cf_error.re_errno = errno;
 			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
 			goto err;
-- 
1.8.3.1

[PATCH 2/3] Record errno value before calling syslog

[Paulo Andrade]

  Unlikely to change, but stay in the safe side.

Signed-off-by: Paulo Andrade <pcpa@gnu.org>
---
 src/clnt_vc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/clnt_vc.c b/src/clnt_vc.c
index 8af7ddd..0da18ca 100644
--- a/src/clnt_vc.c
+++ b/src/clnt_vc.c
@@ -191,10 +191,10 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 	ct = (struct ct_data *)mem_alloc(sizeof (*ct));
 	if ((cl == (CLIENT *)NULL) || (ct == (struct ct_data *)NULL)) {
 		struct rpc_createerr *ce = &get_rpc_createerr();
-		(void) syslog(LOG_ERR, clnt_vc_errstr,
-		    clnt_vc_str, __no_mem_str);
 		ce->cf_stat = RPC_SYSTEMERROR;
 		ce->cf_error.re_errno = errno;
+		(void) syslog(LOG_ERR, clnt_vc_errstr,
+		    clnt_vc_str, __no_mem_str);
 		goto err;
 	}
 	ct->ct_addr.buf = NULL;
-- 
1.8.3.1

[PATCH 3/3] Do not hold a global mutex during connect

[Paulo Andrade]

  A multi-threaded application, connecting to multiple rpc servers,
may dead lock if the connect call stalls on a non responsive server.

Signed-off-by: Paulo Andrade <pcpa@gnu.org>
---
 src/clnt_vc.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/clnt_vc.c b/src/clnt_vc.c
index 0da18ca..0f018d5 100644
--- a/src/clnt_vc.c
+++ b/src/clnt_vc.c
@@ -233,15 +233,16 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 		assert(vc_cv != (cond_t *) NULL);
 
 	/*
-	 * XXX - fvdl connecting while holding a mutex?
+	 * Do not hold mutex during connect
 	 */
+	mutex_unlock(&clnt_fd_lock);
+
 	slen = sizeof ss;
 	if (getpeername(fd, (struct sockaddr *)&ss, &slen) < 0) {
 		if (errno != ENOTCONN) {
 			struct rpc_createerr *ce = &get_rpc_createerr();
 			ce->cf_stat = RPC_SYSTEMERROR;
 			ce->cf_error.re_errno = errno;
-			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
 			goto err;
 		}
@@ -249,12 +250,10 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 			struct rpc_createerr *ce = &get_rpc_createerr();
 			ce->cf_stat = RPC_SYSTEMERROR;
 			ce->cf_error.re_errno = errno;
-			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
 			goto err;
 		}
 	}
-	mutex_unlock(&clnt_fd_lock);
 	if (!__rpc_fd2sockinfo(fd, &si))
 		goto err;
 	thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
-- 
1.8.3.1

Re: [Libtirpc-devel] [PATCH 1/3] Make it clear rpc_createerr is thread safe

[Ian Kent]

On Thu, 2016-05-19 at 12:35 -0300, Paulo Andrade wrote:
>   Avoid hidding it under a macro, and also avoid multiple function
> calls when accessing structure fields.

I like this approach, the use of the macro was confusing to me.
Including that define must always be the case though.

If others agree with this there probably needs to be follow up series with
similar changes for the rest of the source.

I guess the main argument against it is that library users have come to expect
using the macro and could be confused by this change, the exact opposite to the
point of the change.

Thoughts anyone?

> 
> Signed-off-by: Paulo Andrade <pcpa@gnu.org>
> ---
>  src/clnt_vc.c | 17 +++++++++++------
>  1 file changed, 11 insertions(+), 6 deletions(-)
> 
> diff --git a/src/clnt_vc.c b/src/clnt_vc.c
> index a72f9f7..8af7ddd 100644
> --- a/src/clnt_vc.c
> +++ b/src/clnt_vc.c
> @@ -72,6 +72,8 @@
>  #define CMGROUP_MAX    16
>  #define SCM_CREDS      0x03            /* process creds (struct cmsgcred) */
>  
> +#undef rpc_createerr                   /* make it clear it is a thread safe
> variable */
> +
>  /*
>   * Credentials structure, used to verify the identity of a peer
>   * process that has sent us a message. This is allocated by the
> @@ -188,10 +190,11 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  	cl = (CLIENT *)mem_alloc(sizeof (*cl));
>  	ct = (struct ct_data *)mem_alloc(sizeof (*ct));
>  	if ((cl == (CLIENT *)NULL) || (ct == (struct ct_data *)NULL)) {
> +		struct rpc_createerr *ce = &get_rpc_createerr();
>  		(void) syslog(LOG_ERR, clnt_vc_errstr,
>  		    clnt_vc_str, __no_mem_str);
> -		rpc_createerr.cf_stat = RPC_SYSTEMERROR;
> -		rpc_createerr.cf_error.re_errno = errno;
> +		ce->cf_stat = RPC_SYSTEMERROR;
> +		ce->cf_error.re_errno = errno;
>  		goto err;
>  	}
>  	ct->ct_addr.buf = NULL;
> @@ -235,15 +238,17 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  	slen = sizeof ss;
>  	if (getpeername(fd, (struct sockaddr *)&ss, &slen) < 0) {
>  		if (errno != ENOTCONN) {
> -			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
> -			rpc_createerr.cf_error.re_errno = errno;
> +			struct rpc_createerr *ce = &get_rpc_createerr();
> +			ce->cf_stat = RPC_SYSTEMERROR;
> +			ce->cf_error.re_errno = errno;
>  			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
>  		}
>  		if (connect(fd, (struct sockaddr *)raddr->buf, raddr->len) <
> 0){
> -			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
> -			rpc_createerr.cf_error.re_errno = errno;
> +			struct rpc_createerr *ce = &get_rpc_createerr();
> +			ce->cf_stat = RPC_SYSTEMERROR;
> +			ce->cf_error.re_errno = errno;
>  			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;

Re: [Libtirpc-devel] [PATCH 3/3] Do not hold a global mutex during connect

[Ian Kent]

On Thu, 2016-05-19 at 12:35 -0300, Paulo Andrade wrote:
>   A multi-threaded application, connecting to multiple rpc servers,
> may dead lock if the connect call stalls on a non responsive server.

It's occurred to me that the mutex may be held over the connect(2) call to
prevent concurrent calls to connect(2) using the same fd.

That's a race and is a lot harder to deal with.

Comments, thoughts anyone?

I guess it's time to have a look at the connect(2) source ....

> 
> Signed-off-by: Paulo Andrade <pcpa@gnu.org>
> ---
>  src/clnt_vc.c | 7 +++----
>  1 file changed, 3 insertions(+), 4 deletions(-)
> 
> diff --git a/src/clnt_vc.c b/src/clnt_vc.c
> index 0da18ca..0f018d5 100644
> --- a/src/clnt_vc.c
> +++ b/src/clnt_vc.c
> @@ -233,15 +233,16 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  		assert(vc_cv != (cond_t *) NULL);
>  
>  	/*
> -	 * XXX - fvdl connecting while holding a mutex?
> +	 * Do not hold mutex during connect
>  	 */
> +	mutex_unlock(&clnt_fd_lock);
> +
>  	slen = sizeof ss;
>  	if (getpeername(fd, (struct sockaddr *)&ss, &slen) < 0) {
>  		if (errno != ENOTCONN) {
>  			struct rpc_createerr *ce = &get_rpc_createerr();
>  			ce->cf_stat = RPC_SYSTEMERROR;
>  			ce->cf_error.re_errno = errno;
> -			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
>  		}
> @@ -249,12 +250,10 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  			struct rpc_createerr *ce = &get_rpc_createerr();
>  			ce->cf_stat = RPC_SYSTEMERROR;
>  			ce->cf_error.re_errno = errno;
> -			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
>  		}
>  	}
> -	mutex_unlock(&clnt_fd_lock);
>  	if (!__rpc_fd2sockinfo(fd, &si))
>  		goto err;
>  	thr_sigsetmask(SIG_SETMASK, &(mask), NULL);

Re: [Libtirpc-devel] [PATCH 1/3] Make it clear rpc_createerr is thread safe

[Steve Dickson]

On 05/19/2016 11:35 AM, Paulo Andrade wrote:
>   Avoid hidding it under a macro, and also avoid multiple function
> calls when accessing structure fields.
> 
> Signed-off-by: Paulo Andrade <pcpa@gnu.org>
Committed... 

steved.

> ---
>  src/clnt_vc.c | 17 +++++++++++------
>  1 file changed, 11 insertions(+), 6 deletions(-)
> 
> diff --git a/src/clnt_vc.c b/src/clnt_vc.c
> index a72f9f7..8af7ddd 100644
> --- a/src/clnt_vc.c
> +++ b/src/clnt_vc.c
> @@ -72,6 +72,8 @@
>  #define CMGROUP_MAX    16
>  #define SCM_CREDS      0x03            /* process creds (struct cmsgcred) */
>  
> +#undef rpc_createerr                   /* make it clear it is a thread safe variable */
> +
>  /*
>   * Credentials structure, used to verify the identity of a peer
>   * process that has sent us a message. This is allocated by the
> @@ -188,10 +190,11 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  	cl = (CLIENT *)mem_alloc(sizeof (*cl));
>  	ct = (struct ct_data *)mem_alloc(sizeof (*ct));
>  	if ((cl == (CLIENT *)NULL) || (ct == (struct ct_data *)NULL)) {
> +		struct rpc_createerr *ce = &get_rpc_createerr();
>  		(void) syslog(LOG_ERR, clnt_vc_errstr,
>  		    clnt_vc_str, __no_mem_str);
> -		rpc_createerr.cf_stat = RPC_SYSTEMERROR;
> -		rpc_createerr.cf_error.re_errno = errno;
> +		ce->cf_stat = RPC_SYSTEMERROR;
> +		ce->cf_error.re_errno = errno;
>  		goto err;
>  	}
>  	ct->ct_addr.buf = NULL;
> @@ -235,15 +238,17 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  	slen = sizeof ss;
>  	if (getpeername(fd, (struct sockaddr *)&ss, &slen) < 0) {
>  		if (errno != ENOTCONN) {
> -			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
> -			rpc_createerr.cf_error.re_errno = errno;
> +			struct rpc_createerr *ce = &get_rpc_createerr();
> +			ce->cf_stat = RPC_SYSTEMERROR;
> +			ce->cf_error.re_errno = errno;
>  			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
>  		}
>  		if (connect(fd, (struct sockaddr *)raddr->buf, raddr->len) < 0){
> -			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
> -			rpc_createerr.cf_error.re_errno = errno;
> +			struct rpc_createerr *ce = &get_rpc_createerr();
> +			ce->cf_stat = RPC_SYSTEMERROR;
> +			ce->cf_error.re_errno = errno;
>  			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
> 

Re: [Libtirpc-devel] [PATCH 2/3] Record errno value before calling syslog

[Steve Dickson]

On 05/19/2016 11:35 AM, Paulo Andrade wrote:
>   Unlikely to change, but stay in the safe side.
> 
> Signed-off-by: Paulo Andrade <pcpa@gnu.org>
Committed.. 

steved.

> ---
>  src/clnt_vc.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/src/clnt_vc.c b/src/clnt_vc.c
> index 8af7ddd..0da18ca 100644
> --- a/src/clnt_vc.c
> +++ b/src/clnt_vc.c
> @@ -191,10 +191,10 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  	ct = (struct ct_data *)mem_alloc(sizeof (*ct));
>  	if ((cl == (CLIENT *)NULL) || (ct == (struct ct_data *)NULL)) {
>  		struct rpc_createerr *ce = &get_rpc_createerr();
> -		(void) syslog(LOG_ERR, clnt_vc_errstr,
> -		    clnt_vc_str, __no_mem_str);
>  		ce->cf_stat = RPC_SYSTEMERROR;
>  		ce->cf_error.re_errno = errno;
> +		(void) syslog(LOG_ERR, clnt_vc_errstr,
> +		    clnt_vc_str, __no_mem_str);
>  		goto err;
>  	}
>  	ct->ct_addr.buf = NULL;
> 

Re: [Libtirpc-devel] [PATCH 3/3] Do not hold a global mutex during connect

[Steve Dickson]

On 05/19/2016 11:35 AM, Paulo Andrade wrote:
>   A multi-threaded application, connecting to multiple rpc servers,
> may dead lock if the connect call stalls on a non responsive server.
> 
> Signed-off-by: Paulo Andrade <pcpa@gnu.org>
committed... 

steved.
> ---
>  src/clnt_vc.c | 7 +++----
>  1 file changed, 3 insertions(+), 4 deletions(-)
> 
> diff --git a/src/clnt_vc.c b/src/clnt_vc.c
> index 0da18ca..0f018d5 100644
> --- a/src/clnt_vc.c
> +++ b/src/clnt_vc.c
> @@ -233,15 +233,16 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  		assert(vc_cv != (cond_t *) NULL);
>  
>  	/*
> -	 * XXX - fvdl connecting while holding a mutex?
> +	 * Do not hold mutex during connect
>  	 */
> +	mutex_unlock(&clnt_fd_lock);
> +
>  	slen = sizeof ss;
>  	if (getpeername(fd, (struct sockaddr *)&ss, &slen) < 0) {
>  		if (errno != ENOTCONN) {
>  			struct rpc_createerr *ce = &get_rpc_createerr();
>  			ce->cf_stat = RPC_SYSTEMERROR;
>  			ce->cf_error.re_errno = errno;
> -			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
>  		}
> @@ -249,12 +250,10 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
>  			struct rpc_createerr *ce = &get_rpc_createerr();
>  			ce->cf_stat = RPC_SYSTEMERROR;
>  			ce->cf_error.re_errno = errno;
> -			mutex_unlock(&clnt_fd_lock);
>  			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>  			goto err;
>  		}
>  	}
> -	mutex_unlock(&clnt_fd_lock);
>  	if (!__rpc_fd2sockinfo(fd, &si))
>  		goto err;
>  	thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
>