[PATCH v5 1/4] rsockets: fine grained interception mechanism for rsocket preloading

[PATCH v5 1/4] rsockets: fine grained interception mechanism for rsocket preloading

[Sreedhar Kodali]

[-- Attachment #1: Type: text/plain, Size: 13377 bytes --]

Note: This patch is reformatted and is being resent as suggested by Sean
       Also attached the patch file for convenience

 From: Sreedhar Kodali <srkodali-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
Date:   Thu Sep 18 11:33:21 2014 +0530

     By default the R-Sockets pre-loading library intercepts all the 
stream
     and datagram sockets belonging to a launched program processes and 
threads.

     However, distributed application and database servers may require 
fine
     grained interception to ensure that only the processes which are 
listening
     for remote connections on the RDMA transport need to be enabled with 
RDMA
     while remaining can continue to use TCP as before.  This allows 
proper
     communication happening between various server components locally.

     A configuration file based mechanism is introduced to facilitate 
this
     fine grained interception mechanism.  As part of preload 
initialization,
     the configuration file is scanned and an in-memory record store is 
created
     with all the entries found.  When a request is made to intercept a 
socket,
     its attributes are cross checked with stored records to see whether 
we
     should proceed with rsocket switch over.

     Note: Right now, the fine grained interception mechanism is enabled 
only
     for newly created sockets.  Going forward, this can be extened to
     select connections based on the specified host/IP addresses and
     ports as well.

     "preload_config" is the name of the configuration file which should 
exist
     in the default configuration location (usually the full path to this
     configuration file is: 
<install-root>/etc/rdma/rsocket/preload_config)
     of an installed rsocket library.

     The sample format for this configuration file is shown below:

     @#
     @# Sample config file for preloading in a program specific way
     @#
     @# Each line entry should have the following format:
     @#
     @#   prog_name <space> dom_spec <space> type_spec <space> proto_spec
     @#
     @# where,
     @#
     @# prog_name  - program or command name (string without spaces)
     @# dom_spec   - one or more socket domain strings separated by 
commas
     @#            - format: {*|domain,[,domain,...]}
     @#            - '*' means any valid domain
     @#            - valid domains: inet/inet6/ib
     @# type_spec  - one or more socket type strings separated by commas
     @#            - format: {*|type[,type,...]}
     @#            - '*' means any valid type
     @#            - valid types: stream/dgram
     @# proto_spec - one or more socket protocol strings separated by 
commas
     @#            - format: {*|protocol[,protocol,...]}
     @#            - '*' means any valid protocol
     @#            - valid protocols: tcp/udp
     @# <space>    - one ore more tab or space characters
     @#
     @# Note:
     @#  Lines beginning with '#' character are treated as comments.
     @#  Comments at the end of an entry are allowed and should be 
preceded
     @#  by '#' character.
     @#  Blank lines are ignored.
     @
     @progA inet stream tcp # intercept progA's internet stream sockets
     @progB inet6 dgram udp # intercept progB's ipv6 datagram sockets
     @progC * * * # intercept progC's sockets

     Signed-off-by: Sreedhar Kodali <srkodali-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
     Reviewed-by: Pradeep Satyanarayana <pradeeps-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
     ---

diff --git a/src/preload.c b/src/preload.c
index fb2149b..b597616 100644
--- a/src/preload.c
+++ b/src/preload.c
@@ -50,6 +50,8 @@
  #include <netinet/tcp.h>
  #include <unistd.h>
  #include <semaphore.h>
+#include <ctype.h>
+#include <stdlib.h>

  #include <rdma/rdma_cma.h>
  #include <rdma/rdma_verbs.h>
@@ -122,6 +124,217 @@ struct fd_info {
         atomic_t refcnt;
  };

+typedef struct {
+       char *name;
+       uint32_t domain;
+       uint32_t type;
+       uint32_t protocol;
+} config_entry_t;
+
+static config_entry_t *entryp;
+static int16_t nentries;
+static int16_t config_avail;
+extern char *program_invocation_short_name;
+
+/* scan preload configuration file and create
+ * in-memory config store
+ * should be called only once under lock
+ */
+static int scan_preload_config(void)
+{
+       FILE *fp;
+       char line[512];
+       char *lp, *cp, *str1, *str2;
+       char *token, *subtoken, *saveptr1, *saveptr2;
+       int i, j, ret = 0;
+
+       fp = fopen(RS_CONF_DIR "/preload_config", "r");
+       if (fp == NULL) {
+               return -1;
+       }
+
+       while ((lp = fgets(line, sizeof(line), fp)) != NULL) {
+
+               /* trim white space at the beginning of each line */
+               while (*lp != '\0') {
+                       if (isspace(*lp)) {
+                               lp++;
+                               continue;
+                       } else {
+                               break;
+                       }
+               }
+
+               /* skip comment and blank lines */
+               if (*lp == '\0' || *lp == '#') {
+                       continue;
+               }
+
+               /* trim comments and newlines at the end of each line */
+               if ((cp = strpbrk(lp, "#\n")) != NULL) {
+                       *cp = '\0';
+               }
+
+               /* now allocate memory for new configuration entry */
+               entryp = (config_entry_t *) realloc(entryp, (nentries + 
1) *
+                                               sizeof(config_entry_t));
+               if (!entryp) {
+                       ret = -1;
+                       goto scan_done;
+               }
+               memset(entryp + nentries, '\0', sizeof(config_entry_t));
+
+               /* tokenize the retrieved line and parse individual 
fields */
+               for (i = 1, str1 = lp; ; i++, str1 = NULL) {
+                       token = strtok_r(str1, " \t", &saveptr1);
+                       if (token == NULL) {
+                               break;
+                       }
+
+                       /* first field should contain program name */
+                       if (i == 1) {
+                               entryp[nentries].name = (char *) 
malloc((strlen(token) + 1));
+                               if (!entryp[nentries].name) {
+                                       ret = -1;
+                                       goto scan_done;
+                               }
+                               memcpy(entryp[nentries].name, token, 
strlen(token) + 1);
+                               continue;
+                       }
+
+                       /* second field onwards can contain multiple 
entries
+                        * separated by commas
+                        */
+                       for (j = 1, str2 = token; ; j++, str2 = NULL) {
+                               subtoken = strtok_r(str2, ",", 
&saveptr2);
+                               if (subtoken == NULL) {
+                                       break;
+                               }
+
+                               /* second field is socket domain
+                                * rsocket currently recognizes only 
AF_INET, AF_INET6
+                                * and AF_IB domains
+                                * '*' implies all the valid domains
+                                */
+                               if (i == 2) {
+                                       if (*subtoken == '*') {
+                                               entryp[nentries].domain 
|= (1 << AF_INET);
+                                               entryp[nentries].domain 
|= (1 << AF_INET6);
+                                               entryp[nentries].domain 
|= (1 << AF_IB);
+                                               break;
+                                       } else if (strcmp(subtoken, 
"inet6") == 0) {
+                                               entryp[nentries].domain 
|= (1 << AF_INET6);
+                                       } else if (strcmp(subtoken, 
"inet") == 0) {
+                                               entryp[nentries].domain 
|= (1 << AF_INET);
+                                       } else if (strcmp(subtoken, 
"ib") == 0) {
+                                               entryp[nentries].domain 
|= (1 << AF_IB);
+                                       }
+                                       continue;
+                               }
+
+                               /* third field is socket type
+                                * rsocket currently recognizes only 
SOCK_STREAM and
+                                * SOCK_DGRAM types
+                                * '*' implies all the valid types
+                                */
+                               if (i == 3) {
+                                       if (*subtoken == '*') {
+                                               entryp[nentries].type |= 
(1 << SOCK_STREAM);
+                                               entryp[nentries].type |= 
(1 << SOCK_DGRAM);
+                                               break;
+                                       } else if (strcmp(subtoken, 
"stream") == 0) {
+                                               entryp[nentries].type |= 
(1 << SOCK_STREAM);
+                                       } else if (strcmp(subtoken, 
"dgram") == 0) {
+                                               entryp[nentries].type |= 
(1 << SOCK_DGRAM);
+                                       }
+                                       continue;
+                               }
+
+                               /* fourth field is socket protocol
+                                * rsocket currently recgonizes only 
IPPROTO_TCP and
+                                * IPPROTO_UDP protocols
+                                * '*' implies all the valid protocols
+                                */
+                               if (i == 4) {
+                                       if (*subtoken == '*') {
+                                               
entryp[nentries].protocol |= (1 << IPPROTO_TCP);
+                                               
entryp[nentries].protocol |= (1 << IPPROTO_UDP);
+                                               break;
+                                       } else if (strcmp(subtoken, 
"tcp") == 0) {
+                                               
entryp[nentries].protocol |= (1 << IPPROTO_TCP);
+                                       } else if (strcmp(subtoken, 
"udp") == 0) {
+                                               
entryp[nentries].protocol |= (1 << IPPROTO_UDP);
+                                       }
+                                       continue;
+                               }
+                       }
+               }
+               nentries += 1;
+       }
+
+scan_done:
+       fclose(fp);
+       return ret;
+}
+
+/* free in-memory config store
+ * should be called only once during finalization
+ */
+static void free_preload_config(void)
+{
+       int i;
+
+       if (entryp) {
+               for (i = 0; i < nentries; i++) {
+                       if (entryp[i].name) {
+                               free(entryp[i].name);
+                       }
+               }
+               free(entryp);
+       }
+
+       return;
+}
+
+/* check whether interception is required for this socket
+ * compares the provided attributes with that available in the 
in-memory
+ * data store for the current process
+ * sets-up in-memory config store if it's already not done
+ */
+static int intercept_socket(int domain, int type, int protocol)
+{
+       int i;
+
+       /* locate the config entry */
+       for (i = 0; i < nentries; i++) {
+               if (strncmp(entryp[i].name, 
program_invocation_short_name,
+                                       strlen(entryp[i].name)) == 0) {
+                       break;
+               }
+       }
+       if (i == nentries) {
+               return 0;
+       }
+
+       /* match domain field */
+       if (!(entryp[i].domain & (1 << domain))) {
+               return 0;
+       }
+
+       /* match type field */
+       if (!(entryp[i].type & (1 << type))) {
+               return 0;
+       }
+
+       /* match protocol field only if protocol is specified */
+       if (protocol && !(entryp[i].protocol & (1 << protocol))) {
+               return 0;
+       }
+
+       /* entry matched */
+       return 1;
+}
+
  static int fd_open(void)
  {
         struct fd_info *fdi;
@@ -308,6 +521,14 @@ static void init_preload(void)
         rs.fcntl = dlsym(RTLD_DEFAULT, "rfcntl");

         getenv_options();
+
+       if (scan_preload_config() == 0) {
+               config_avail = 1;
+       }
+       if (entryp) {
+               atexit(free_preload_config);
+       }
+
         init = 1;
  out:
         pthread_mutex_unlock(&mut);
@@ -404,10 +625,15 @@ int socket(int domain, int type, int protocol)
         static __thread int recursive;
         int index, ret;

+       init_preload();
+
+       if (config_avail && intercept_socket(domain, type, protocol) == 
0) {
+               goto real;
+       }
+
         if (recursive)
                 goto real;

-       init_preload();
         index = fd_open();
         if (index < 0)
                 return index;

[-- Attachment #2: librdmacm_src_preload_fine_grained_20140918.patch1 --]
[-- Type: text/plain, Size: 9590 bytes --]

commit 89afd8cfcd1347f793ed7928f643213b6dbd4599
Author: Sreedhar Kodali <srkodali@linux.vnet.ibm.com>
Date:   Thu Sep 18 11:33:21 2014 +0530

    By default the R-Sockets pre-loading library intercepts all the stream
    and datagram sockets belonging to a launched program processes and threads.
    
    However, distributed application and database servers may require fine
    grained interception to ensure that only the processes which are listening
    for remote connections on the RDMA transport need to be enabled with RDMA
    while remaining can continue to use TCP as before.  This allows proper
    communication happening between various server components locally.
    
    A configuration file based mechanism is introduced to facilitate this
    fine grained interception mechanism.  As part of preload initialization,
    the configuration file is scanned and an in-memory record store is created
    with all the entries found.  When a request is made to intercept a socket,
    its attributes are cross checked with stored records to see whether we
    should proceed with rsocket switch over.
    
    Note: Right now, the fine grained interception mechanism is enabled only
    for newly created sockets.  Going forward, this can be extened to
    select connections based on the specified host/IP addresses and
    ports as well.
    
    "preload_config" is the name of the configuration file which should exist
    in the default configuration location (usually the full path to this
    configuration file is: <install-root>/etc/rdma/rsocket/preload_config)
    of an installed rsocket library.
    
    The sample format for this configuration file is shown below:
    
    @#
    @# Sample config file for preloading in a program specific way
    @#
    @# Each line entry should have the following format:
    @#
    @#   prog_name <space> dom_spec <space> type_spec <space> proto_spec
    @#
    @# where,
    @#
    @# prog_name  - program or command name (string without spaces)
    @# dom_spec   - one or more socket domain strings separated by commas
    @#            - format: {*|domain,[,domain,...]}
    @#            - '*' means any valid domain
    @#            - valid domains: inet/inet6/ib
    @# type_spec  - one or more socket type strings separated by commas
    @#            - format: {*|type[,type,...]}
    @#            - '*' means any valid type
    @#            - valid types: stream/dgram
    @# proto_spec - one or more socket protocol strings separated by commas
    @#            - format: {*|protocol[,protocol,...]}
    @#            - '*' means any valid protocol
    @#            - valid protocols: tcp/udp
    @# <space>    - one ore more tab or space characters
    @#
    @# Note:
    @#  Lines beginning with '#' character are treated as comments.
    @#  Comments at the end of an entry are allowed and should be preceded
    @#  by '#' character.
    @#  Blank lines are ignored.
    @
    @progA inet stream tcp # intercept progA's internet stream sockets
    @progB inet6 dgram udp # intercept progB's ipv6 datagram sockets
    @progC * * * # intercept progC's sockets
    
    Signed-off-by: Sreedhar Kodali <srkodali@linux.vnet.ibm.com>
    Reviewed-by: Pradeep Satyanarayana <pradeeps@linux.vnet.ibm.com>
    ---

diff --git a/src/preload.c b/src/preload.c
index fb2149b..b597616 100644
--- a/src/preload.c
+++ b/src/preload.c
@@ -50,6 +50,8 @@
 #include <netinet/tcp.h>
 #include <unistd.h>
 #include <semaphore.h>
+#include <ctype.h>
+#include <stdlib.h>
 
 #include <rdma/rdma_cma.h>
 #include <rdma/rdma_verbs.h>
@@ -122,6 +124,217 @@ struct fd_info {
 	atomic_t refcnt;
 };
 
+typedef struct {
+	char *name;
+	uint32_t domain;
+	uint32_t type;
+	uint32_t protocol;
+} config_entry_t;
+
+static config_entry_t *entryp;
+static int16_t nentries;
+static int16_t config_avail;
+extern char *program_invocation_short_name;
+
+/* scan preload configuration file and create
+ * in-memory config store
+ * should be called only once under lock
+ */
+static int scan_preload_config(void)
+{
+	FILE *fp;
+	char line[512];
+	char *lp, *cp, *str1, *str2;
+	char *token, *subtoken, *saveptr1, *saveptr2;
+	int i, j, ret = 0;
+
+	fp = fopen(RS_CONF_DIR "/preload_config", "r");
+	if (fp == NULL) {
+		return -1;
+	}
+
+	while ((lp = fgets(line, sizeof(line), fp)) != NULL) {
+
+		/* trim white space at the beginning of each line */
+		while (*lp != '\0') {
+			if (isspace(*lp)) {
+				lp++;
+				continue;
+			} else {
+				break;
+			}
+		}
+
+		/* skip comment and blank lines */
+		if (*lp == '\0' || *lp == '#') {
+			continue;
+		}
+
+		/* trim comments and newlines at the end of each line */
+		if ((cp = strpbrk(lp, "#\n")) != NULL) {
+			*cp = '\0';
+		}
+
+		/* now allocate memory for new configuration entry */
+		entryp = (config_entry_t *) realloc(entryp, (nentries + 1) *
+						sizeof(config_entry_t));
+		if (!entryp) {
+			ret = -1;
+			goto scan_done;
+		}
+		memset(entryp + nentries, '\0', sizeof(config_entry_t));
+
+		/* tokenize the retrieved line and parse individual fields */
+		for (i = 1, str1 = lp; ; i++, str1 = NULL) {
+			token = strtok_r(str1, " \t", &saveptr1);
+			if (token == NULL) {
+				break;
+			}
+
+			/* first field should contain program name */
+			if (i == 1) {
+				entryp[nentries].name = (char *) malloc((strlen(token) + 1));
+				if (!entryp[nentries].name) {
+					ret = -1;
+					goto scan_done;
+				}
+				memcpy(entryp[nentries].name, token, strlen(token) + 1);
+				continue;
+			}
+
+			/* second field onwards can contain multiple entries
+			 * separated by commas
+			 */
+			for (j = 1, str2 = token; ; j++, str2 = NULL) {
+				subtoken = strtok_r(str2, ",", &saveptr2);
+				if (subtoken == NULL) {
+					break;
+				}
+
+				/* second field is socket domain
+				 * rsocket currently recognizes only AF_INET, AF_INET6
+				 * and AF_IB domains
+				 * '*' implies all the valid domains
+				 */
+				if (i == 2) {
+					if (*subtoken == '*') {
+						entryp[nentries].domain |= (1 << AF_INET);
+						entryp[nentries].domain |= (1 << AF_INET6);
+						entryp[nentries].domain |= (1 << AF_IB);
+						break;
+					} else if (strcmp(subtoken, "inet6") == 0) {
+						entryp[nentries].domain |= (1 << AF_INET6);
+					} else if (strcmp(subtoken, "inet") == 0) {
+						entryp[nentries].domain |= (1 << AF_INET);
+					} else if (strcmp(subtoken, "ib") == 0) {
+						entryp[nentries].domain |= (1 << AF_IB);
+					}
+					continue;
+				}
+
+				/* third field is socket type
+				 * rsocket currently recognizes only SOCK_STREAM and
+				 * SOCK_DGRAM types
+				 * '*' implies all the valid types
+				 */
+				if (i == 3) {
+					if (*subtoken == '*') {
+						entryp[nentries].type |= (1 << SOCK_STREAM);
+						entryp[nentries].type |= (1 << SOCK_DGRAM);
+						break;
+					} else if (strcmp(subtoken, "stream") == 0) {
+						entryp[nentries].type |= (1 << SOCK_STREAM);
+					} else if (strcmp(subtoken, "dgram") == 0) {
+						entryp[nentries].type |= (1 << SOCK_DGRAM);
+					}
+					continue;
+				}
+
+				/* fourth field is socket protocol
+				 * rsocket currently recgonizes only IPPROTO_TCP and
+				 * IPPROTO_UDP protocols
+				 * '*' implies all the valid protocols
+				 */
+				if (i == 4) {
+					if (*subtoken == '*') {
+						entryp[nentries].protocol |= (1 << IPPROTO_TCP);
+						entryp[nentries].protocol |= (1 << IPPROTO_UDP);
+						break;
+					} else if (strcmp(subtoken, "tcp") == 0) {
+						entryp[nentries].protocol |= (1 << IPPROTO_TCP);
+					} else if (strcmp(subtoken, "udp") == 0) {
+						entryp[nentries].protocol |= (1 << IPPROTO_UDP);
+					}
+					continue;
+				}
+			}
+		}
+		nentries += 1;
+	}
+
+scan_done:
+	fclose(fp);
+	return ret;
+}
+
+/* free in-memory config store
+ * should be called only once during finalization
+ */
+static void free_preload_config(void)
+{
+	int i;
+
+	if (entryp) {
+		for (i = 0; i < nentries; i++) {
+			if (entryp[i].name) {
+				free(entryp[i].name);
+			}
+		}
+		free(entryp);
+	}
+
+	return;
+}
+
+/* check whether interception is required for this socket
+ * compares the provided attributes with that available in the in-memory
+ * data store for the current process
+ * sets-up in-memory config store if it's already not done
+ */
+static int intercept_socket(int domain, int type, int protocol)
+{
+	int i;
+
+	/* locate the config entry */
+	for (i = 0; i < nentries; i++) {
+		if (strncmp(entryp[i].name, program_invocation_short_name,
+					strlen(entryp[i].name)) == 0) {
+			break;
+		}
+	}
+	if (i == nentries) {
+		return 0;
+	}
+
+	/* match domain field */
+	if (!(entryp[i].domain & (1 << domain))) {
+		return 0;
+	}
+
+	/* match type field */
+	if (!(entryp[i].type & (1 << type))) {
+		return 0;
+	}
+
+	/* match protocol field only if protocol is specified */
+	if (protocol && !(entryp[i].protocol & (1 << protocol))) {
+		return 0;
+	}
+
+	/* entry matched */
+	return 1;
+}
+
 static int fd_open(void)
 {
 	struct fd_info *fdi;
@@ -308,6 +521,14 @@ static void init_preload(void)
 	rs.fcntl = dlsym(RTLD_DEFAULT, "rfcntl");
 
 	getenv_options();
+	
+	if (scan_preload_config() == 0) {
+		config_avail = 1;
+	}
+	if (entryp) {
+		atexit(free_preload_config);
+	}
+
 	init = 1;
 out:
 	pthread_mutex_unlock(&mut);
@@ -404,10 +625,15 @@ int socket(int domain, int type, int protocol)
 	static __thread int recursive;
 	int index, ret;
 
+	init_preload();
+
+	if (config_avail && intercept_socket(domain, type, protocol) == 0) {
+		goto real;
+	}
+
 	if (recursive)
 		goto real;
 
-	init_preload();
 	index = fd_open();
 	if (index < 0)
 		return index;

RE: [PATCH v5 1/4] rsockets: fine grained interception mechanism for rsocket preloading

[Hefty, Sean]

From: Sreedhar Kodali <srkodali-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>

By default the R-Sockets pre-loading library intercepts all
the stream and datagram sockets belonging to a launched
program processes and threads.

However, distributed application and database servers may
require fine grained interception to ensure that only the
processes which are listening for remote connections on the
RDMA transport need to be enabled with RDMA while remaining
can continue to use TCP as before.  This allows proper
communication happening between various server components locally.

A configuration file based mechanism is introduced to facilitate
this fine grained interception mechanism.  As part of preload
initialization, the configuration file is scanned and an
in-memory record store is created with all the entries found.
When a request is made to intercept a socket, its attributes
are cross checked with stored records to see whether we
should proceed with rsocket switch over.

Note: Right now, the fine grained interception mechanism is
enabled only for newly created sockets.  Going forward,
this can be extened to select connections based on the
specified host/IP addresses and ports as well.

"preload_config" is the name of the configuration file which
should exist in the default configuration location
(usually the full path to this configuration file is:
<install-root>/etc/rdma/rsocket/preload_config)
of an installed rsocket library.

The sample format for this configuration file is shown below:

# Sample config file for preloading in a program specific way
#
# Each line entry should have the following format:
#
#   program domain type protocol
#
# where,
#
# program    - program or command name (string without spaces)
# domain     - the socket domain: AF_INET / AF_INET6 / AF_IB
# type       - the socket type: SOCK_STREAM / SOCK_DGRAM
# protocol   - the socket protocol: IPPROTO_TCP / IPPROTO_UDP
#
# The wildcard value of '*' is supported for any
#
# Note:
#  Lines beginning with '#' character are treated as comments.

Signed-off-by: Sreedhar Kodali <srkodali-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
Reviewed-by: Pradeep Satyanarayana <pradeeps-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
I made several adjustments to the submitted patch.  Please
verify that these work for you.  Changes from v5:

- Simplified input file format slightly.
- Removed typedef
- Rename entryp and removed unneeded variable
- Replaced token parsing with sscanf
- Added wildcard support for the program name
- Wildcard values now stored as 0
- Enhanced checks for domain, type, and protocol strings
- Fixed realloc error handling
- Simplified free_config implementation
- Added support for app passing in 0 for protocol

 src/preload.c |  138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 136 insertions(+), 2 deletions(-)

diff --git a/src/preload.c b/src/preload.c
index fb2149b..05ac48f 100644
--- a/src/preload.c
+++ b/src/preload.c
@@ -50,6 +50,9 @@
 #include <netinet/tcp.h>
 #include <unistd.h>
 #include <semaphore.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdio.h>
 
 #include <rdma/rdma_cma.h>
 #include <rdma/rdma_verbs.h>
@@ -122,6 +125,135 @@ struct fd_info {
 	atomic_t refcnt;
 };
 
+struct config_entry {
+	char *name;
+	int domain;
+	int type;
+	int protocol;
+};
+
+static struct config_entry *config;
+static int config_cnt;
+extern char *program_invocation_short_name;
+
+
+static void free_config(void)
+{
+	while (config_cnt)
+		free(config[--config_cnt].name);
+
+	free(config);
+}
+
+/*
+ * Config file format:
+ * # Starting '#' indicates comment
+ * # wild card values are supported using '*'
+ * # domain - *, INET, INET6, IB
+ * # type - *, STREAM, DGRAM
+ * # protocol - *, TCP, UDP
+ * program_name domain type protocol
+ */
+static void scan_config(void)
+{
+	struct config_entry *new_config;
+	FILE *fp;
+	char line[120], prog[64], dom[16], type[16], proto[16];
+
+	fp = fopen(RS_CONF_DIR "/preload_config", "r");
+	if (!fp)
+		return;
+
+	while (fgets(line, sizeof(line), fp)) {
+		if (line[0] == '#')
+			continue;
+
+		if (sscanf(line, "%64s%16s%16s%16s", prog, dom, type, proto) != 4)
+			continue;
+
+		new_config = realloc(config, (config_cnt + 1) *
+					     sizeof(struct config_entry));
+		if (!new_config)
+			break;
+
+		config = new_config;
+		memset(&config[config_cnt], 0, sizeof(struct config_entry));
+
+		if (!strcasecmp(dom, "INET") ||
+		    !strcasecmp(dom, "AF_INET") ||
+		    !strcasecmp(dom, "PF_INET")) {
+			config[config_cnt].domain = AF_INET;
+		} else if (!strcasecmp(dom, "INET6") ||
+			   !strcasecmp(dom, "AF_INET6") ||
+			   !strcasecmp(dom, "PF_INET6")) {
+			config[config_cnt].domain = AF_INET6;
+		} else if (!strcasecmp(dom, "IB") ||
+			   !strcasecmp(dom, "AF_IB") ||
+			   !strcasecmp(dom, "PF_IB")) {
+			config[config_cnt].domain = AF_IB;
+		} else if (strcmp(dom, "*")) {
+			continue;
+		}
+
+		if (!strcasecmp(type, "STREAM") ||
+		    !strcasecmp(type, "SOCK_STREAM")) {
+			config[config_cnt].type = SOCK_STREAM;
+		} else if (!strcasecmp(type, "DGRAM") ||
+			   !strcasecmp(type, "SOCK_DGRAM")) {
+			config[config_cnt].type = SOCK_DGRAM;
+		} else if (strcmp(type, "*")) {
+			continue;
+		}
+
+		if (!strcasecmp(proto, "TCP") ||
+		    !strcasecmp(proto, "IPPROTO_TCP")) {
+			config[config_cnt].protocol = IPPROTO_TCP;
+		} else if (!strcasecmp(proto, "UDP") ||
+			   !strcasecmp(proto, "IPPROTO_UDP")) {
+			config[config_cnt].protocol = IPPROTO_UDP;
+		} else if (strcmp(proto, "*")) {
+			continue;
+		}
+
+		if (strcmp(prog, "*")) {
+		    if (!(config[config_cnt].name = strdup(prog)))
+			    continue;
+		}
+
+		config_cnt++;
+	}
+
+	fclose(fp);
+	if (config_cnt)
+		atexit(free_config);
+}
+
+static int intercept_socket(int domain, int type, int protocol)
+{
+	int i;
+
+	if (!config_cnt)
+		return 1;
+
+	if (!protocol) {
+		if (type == SOCK_STREAM)
+			protocol = IPPROTO_TCP;
+		else if (type == SOCK_DGRAM)
+			protocol = IPPROTO_UDP;
+	}
+
+	for (i = 0; i < config_cnt; i++) {
+		if ((!config[i].name ||
+		     !strcasecmp(config[i].name, program_invocation_short_name)) &&
+		    (!config[i].domain || config[i].domain == domain) &&
+		    (!config[i].type || config[i].type == type) &&
+		    (!config[i].protocol || config[i].protocol == protocol))
+			return 1;
+	}
+
+	return 0;
+}
+
 static int fd_open(void)
 {
 	struct fd_info *fdi;
@@ -308,6 +440,7 @@ static void init_preload(void)
 	rs.fcntl = dlsym(RTLD_DEFAULT, "rfcntl");
 
 	getenv_options();
+	scan_config();
 	init = 1;
 out:
 	pthread_mutex_unlock(&mut);
@@ -404,10 +537,11 @@ int socket(int domain, int type, int protocol)
 	static __thread int recursive;
 	int index, ret;
 
-	if (recursive)
+	init_preload();
+
+	if (recursive || !intercept_socket(domain, type, protocol))
 		goto real;
 
-	init_preload();
 	index = fd_open();
 	if (index < 0)
 		return index;


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v5 1/4] rsockets: fine grained interception mechanism for rsocket preloading

[Sreedhar Kodali]

Hi Sean,

Thanks for the review and modifications.  Except one minor
correction the changes are working fine.

Instead of strcasecmp() we should be using strncasecmp()
for comparing program names in intercept_socket() so the
interception should work even if the program prefix is
specified.  It's quite common in distributed server
environments to find several related processes with the
same prefix.  This minor change allows the interception of
all or one of them at user discretion.

I have sent v6 of the patch separately with the above
minor change.  Please have a look at it.

Thank You.

- Sreedhar

On 2014-09-26 04:29, Hefty, Sean wrote:
> From: Sreedhar Kodali <srkodali-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
> 
> By default the R-Sockets pre-loading library intercepts all
> the stream and datagram sockets belonging to a launched
> program processes and threads.
> 
> However, distributed application and database servers may
> require fine grained interception to ensure that only the
> processes which are listening for remote connections on the
> RDMA transport need to be enabled with RDMA while remaining
> can continue to use TCP as before.  This allows proper
> communication happening between various server components locally.
> 
> A configuration file based mechanism is introduced to facilitate
> this fine grained interception mechanism.  As part of preload
> initialization, the configuration file is scanned and an
> in-memory record store is created with all the entries found.
> When a request is made to intercept a socket, its attributes
> are cross checked with stored records to see whether we
> should proceed with rsocket switch over.
> 
> Note: Right now, the fine grained interception mechanism is
> enabled only for newly created sockets.  Going forward,
> this can be extened to select connections based on the
> specified host/IP addresses and ports as well.
> 
> "preload_config" is the name of the configuration file which
> should exist in the default configuration location
> (usually the full path to this configuration file is:
> <install-root>/etc/rdma/rsocket/preload_config)
> of an installed rsocket library.
> 
> The sample format for this configuration file is shown below:
> 
> # Sample config file for preloading in a program specific way
> #
> # Each line entry should have the following format:
> #
> #   program domain type protocol
> #
> # where,
> #
> # program    - program or command name (string without spaces)
> # domain     - the socket domain: AF_INET / AF_INET6 / AF_IB
> # type       - the socket type: SOCK_STREAM / SOCK_DGRAM
> # protocol   - the socket protocol: IPPROTO_TCP / IPPROTO_UDP
> #
> # The wildcard value of '*' is supported for any
> #
> # Note:
> #  Lines beginning with '#' character are treated as comments.
> 
> Signed-off-by: Sreedhar Kodali <srkodali-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
> Reviewed-by: Pradeep Satyanarayana <pradeeps-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
> Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> ---
> I made several adjustments to the submitted patch.  Please
> verify that these work for you.  Changes from v5:
> 
> - Simplified input file format slightly.
> - Removed typedef
> - Rename entryp and removed unneeded variable
> - Replaced token parsing with sscanf
> - Added wildcard support for the program name
> - Wildcard values now stored as 0
> - Enhanced checks for domain, type, and protocol strings
> - Fixed realloc error handling
> - Simplified free_config implementation
> - Added support for app passing in 0 for protocol
> 
>  src/preload.c |  138 
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 files changed, 136 insertions(+), 2 deletions(-)
> 
> diff --git a/src/preload.c b/src/preload.c
> index fb2149b..05ac48f 100644
> --- a/src/preload.c
> +++ b/src/preload.c
> @@ -50,6 +50,9 @@
>  #include <netinet/tcp.h>
>  #include <unistd.h>
>  #include <semaphore.h>
> +#include <ctype.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> 
>  #include <rdma/rdma_cma.h>
>  #include <rdma/rdma_verbs.h>
> @@ -122,6 +125,135 @@ struct fd_info {
>  	atomic_t refcnt;
>  };
> 
> +struct config_entry {
> +	char *name;
> +	int domain;
> +	int type;
> +	int protocol;
> +};
> +
> +static struct config_entry *config;
> +static int config_cnt;
> +extern char *program_invocation_short_name;
> +
> +
> +static void free_config(void)
> +{
> +	while (config_cnt)
> +		free(config[--config_cnt].name);
> +
> +	free(config);
> +}
> +
> +/*
> + * Config file format:
> + * # Starting '#' indicates comment
> + * # wild card values are supported using '*'
> + * # domain - *, INET, INET6, IB
> + * # type - *, STREAM, DGRAM
> + * # protocol - *, TCP, UDP
> + * program_name domain type protocol
> + */
> +static void scan_config(void)
> +{
> +	struct config_entry *new_config;
> +	FILE *fp;
> +	char line[120], prog[64], dom[16], type[16], proto[16];
> +
> +	fp = fopen(RS_CONF_DIR "/preload_config", "r");
> +	if (!fp)
> +		return;
> +
> +	while (fgets(line, sizeof(line), fp)) {
> +		if (line[0] == '#')
> +			continue;
> +
> +		if (sscanf(line, "%64s%16s%16s%16s", prog, dom, type, proto) != 4)
> +			continue;
> +
> +		new_config = realloc(config, (config_cnt + 1) *
> +					     sizeof(struct config_entry));
> +		if (!new_config)
> +			break;
> +
> +		config = new_config;
> +		memset(&config[config_cnt], 0, sizeof(struct config_entry));
> +
> +		if (!strcasecmp(dom, "INET") ||
> +		    !strcasecmp(dom, "AF_INET") ||
> +		    !strcasecmp(dom, "PF_INET")) {
> +			config[config_cnt].domain = AF_INET;
> +		} else if (!strcasecmp(dom, "INET6") ||
> +			   !strcasecmp(dom, "AF_INET6") ||
> +			   !strcasecmp(dom, "PF_INET6")) {
> +			config[config_cnt].domain = AF_INET6;
> +		} else if (!strcasecmp(dom, "IB") ||
> +			   !strcasecmp(dom, "AF_IB") ||
> +			   !strcasecmp(dom, "PF_IB")) {
> +			config[config_cnt].domain = AF_IB;
> +		} else if (strcmp(dom, "*")) {
> +			continue;
> +		}
> +
> +		if (!strcasecmp(type, "STREAM") ||
> +		    !strcasecmp(type, "SOCK_STREAM")) {
> +			config[config_cnt].type = SOCK_STREAM;
> +		} else if (!strcasecmp(type, "DGRAM") ||
> +			   !strcasecmp(type, "SOCK_DGRAM")) {
> +			config[config_cnt].type = SOCK_DGRAM;
> +		} else if (strcmp(type, "*")) {
> +			continue;
> +		}
> +
> +		if (!strcasecmp(proto, "TCP") ||
> +		    !strcasecmp(proto, "IPPROTO_TCP")) {
> +			config[config_cnt].protocol = IPPROTO_TCP;
> +		} else if (!strcasecmp(proto, "UDP") ||
> +			   !strcasecmp(proto, "IPPROTO_UDP")) {
> +			config[config_cnt].protocol = IPPROTO_UDP;
> +		} else if (strcmp(proto, "*")) {
> +			continue;
> +		}
> +
> +		if (strcmp(prog, "*")) {
> +		    if (!(config[config_cnt].name = strdup(prog)))
> +			    continue;
> +		}
> +
> +		config_cnt++;
> +	}
> +
> +	fclose(fp);
> +	if (config_cnt)
> +		atexit(free_config);
> +}
> +
> +static int intercept_socket(int domain, int type, int protocol)
> +{
> +	int i;
> +
> +	if (!config_cnt)
> +		return 1;
> +
> +	if (!protocol) {
> +		if (type == SOCK_STREAM)
> +			protocol = IPPROTO_TCP;
> +		else if (type == SOCK_DGRAM)
> +			protocol = IPPROTO_UDP;
> +	}
> +
> +	for (i = 0; i < config_cnt; i++) {
> +		if ((!config[i].name ||
> +		     !strcasecmp(config[i].name, program_invocation_short_name)) &&
> +		    (!config[i].domain || config[i].domain == domain) &&
> +		    (!config[i].type || config[i].type == type) &&
> +		    (!config[i].protocol || config[i].protocol == protocol))
> +			return 1;
> +	}
> +
> +	return 0;
> +}
> +
>  static int fd_open(void)
>  {
>  	struct fd_info *fdi;
> @@ -308,6 +440,7 @@ static void init_preload(void)
>  	rs.fcntl = dlsym(RTLD_DEFAULT, "rfcntl");
> 
>  	getenv_options();
> +	scan_config();
>  	init = 1;
>  out:
>  	pthread_mutex_unlock(&mut);
> @@ -404,10 +537,11 @@ int socket(int domain, int type, int protocol)
>  	static __thread int recursive;
>  	int index, ret;
> 
> -	if (recursive)
> +	init_preload();
> +
> +	if (recursive || !intercept_socket(domain, type, protocol))
>  		goto real;
> 
> -	init_preload();
>  	index = fd_open();
>  	if (index < 0)
>  		return index;

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html