[PATCH rdma-core 0/4] Allow systemd to also setup the RDMA_LS socket for ibacm

[PATCH rdma-core 0/4] Allow systemd to also setup the RDMA_LS socket for ibacm

[Jason Gunthorpe]

This expands the socket activation for ibacm to include the kernel facing
listening netlink socket. This helps avoid racing during bootup by
allowing the kernel to block resolution until ibacm is started during
earlier boot.

Also a small clean up of F_SETFL

https://github.com/linux-rdma/rdma-core/pull/198

Jason Gunthorpe (4):
  Add set_fd_nonblock to util
  srp_daemon: Use pipe2 instead of pip&fcntl
  Use set_fd_nonblock in instead of open coding F_SETFL
  ibacm: Allow systemd to also setup the RDMA_LS socket for ibacm

 ibacm/ibacm.socket      |  4 +++
 ibacm/src/acm.c         | 81 ++++++++++++++++++++++++++++++++++++-------------
 librdmacm/rsocket.c     |  7 +++--
 srp_daemon/srp_daemon.c | 19 +-----------
 util/CMakeLists.txt     |  5 ++-
 util/dummy.c            |  0
 util/util.c             | 22 ++++++++++++++
 util/util.h             |  4 +++
 8 files changed, 97 insertions(+), 45 deletions(-)
 delete mode 100644 util/dummy.c
 create mode 100644 util/util.c

-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH rdma-core 1/4] Add set_fd_nonblock to util

[Jason Gunthorpe]

This wsa often open coded without the GETFL. GETFL is techinically
required by POSIX as we can't know for sure that the flags are 0
(although today, on Linux, they are for sockets).

Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
---
 util/CMakeLists.txt |  5 ++---
 util/dummy.c        |  0
 util/util.c         | 22 ++++++++++++++++++++++
 util/util.h         |  4 ++++
 4 files changed, 28 insertions(+), 3 deletions(-)
 delete mode 100644 util/dummy.c
 create mode 100644 util/util.c

diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt
index 2ba6a7c79e222c..57d9109653da82 100644
--- a/util/CMakeLists.txt
+++ b/util/CMakeLists.txt
@@ -4,9 +4,8 @@ publish_internal_headers(util
   util.h
   )
 
-# The empty dummy.c is only needed so that cmake always has something to build
-# into the library.
-set(C_FILES dummy.c)
+set(C_FILES
+  util.c)
 
 if (HAVE_COHERENT_DMA)
   publish_internal_headers(util
diff --git a/util/dummy.c b/util/dummy.c
deleted file mode 100644
index e69de29bb2d1d6..00000000000000
diff --git a/util/util.c b/util/util.c
new file mode 100644
index 00000000000000..8c5f8f1ae1989a
--- /dev/null
+++ b/util/util.c
@@ -0,0 +1,22 @@
+/* GPLv2 or OpenIB.org BSD (MIT) See COPYING file */
+#include <util/util.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+int set_fd_nonblock(int fd, bool nonblock)
+{
+	int val;
+
+	val = fcntl(fd, F_GETFL);
+	if (val == -1)
+		return -1;
+
+	if (nonblock)
+		val |= O_NONBLOCK;
+	else
+		val &= ~(unsigned int)(O_NONBLOCK);
+
+	if (fcntl(fd, F_SETFL, val) == -1)
+		return -1;
+	return 0;
+}
diff --git a/util/util.h b/util/util.h
index c3e59554df210c..cbf0deca2dde7b 100644
--- a/util/util.h
+++ b/util/util.h
@@ -2,6 +2,8 @@
 #ifndef UTIL_UTIL_H
 #define UTIL_UTIL_H
 
+#include <stdbool.h>
+
 /* Return true if the snprintf succeeded, false if there was truncation or
  * error */
 #define check_snprintf(buf, len, fmt, ...)                                     \
@@ -16,4 +18,6 @@
 	 ((a)->tv_nsec CMP (b)->tv_nsec) :	\
 	 ((a)->tv_sec CMP (b)->tv_sec))
 
+int set_fd_nonblock(int fd, bool nonblock);
+
 #endif
-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH rdma-core 2/4] srp_daemon: Use pipe2 instead of pip&fcntl

[Jason Gunthorpe]

Just make a nonblocking pipe directly.

Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
---
 srp_daemon/srp_daemon.c | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/srp_daemon/srp_daemon.c b/srp_daemon/srp_daemon.c
index 94d52673dc0fb6..2465ccd9e74f07 100644
--- a/srp_daemon/srp_daemon.c
+++ b/srp_daemon/srp_daemon.c
@@ -1998,25 +1998,12 @@ static int setup_wakeup_fd(void)
 {
 	struct sigaction sa = {};
 	int ret;
-	int i;
-	int flags;
 
-	ret = pipe(wakeup_pipe);
+	ret = pipe2(wakeup_pipe, O_NONBLOCK | O_CLOEXEC);
 	if (ret < 0) {
 		pr_err("could not create pipe\n");
 		return -1;
 	}
-	for (i = 0; i < 2; i++) {
-		flags = fcntl(wakeup_pipe[i], F_GETFL);
-		if (flags < 0) {
-			pr_err("fcntl F_GETFL failed for %d\n", wakeup_pipe[i]);
-			goto close_pipe;
-		}
-		if (fcntl(wakeup_pipe[i], F_SETFL, flags | O_NONBLOCK) < 0) {
-			pr_err("fcntl F_SETFL failed for %d\n", wakeup_pipe[i]);
-			goto close_pipe;
-		}
-	}
 
 	sigemptyset(&sa.sa_mask);
 	sa.sa_handler = signal_handler;
@@ -2024,10 +2011,6 @@ static int setup_wakeup_fd(void)
 	sigaction(SIGTERM, &sa, NULL);
 	sigaction(SRP_CATAS_ERR, &sa, NULL);
 	return 0;
-
-close_pipe:
-	cleanup_wakeup_fd();
-	return -1;
 }
 
 static int ibsrpdm(int argc, char *argv[])
-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH rdma-core 3/4] Use set_fd_nonblock in instead of open coding F_SETFL

[Jason Gunthorpe]

In all places.

Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
---
 ibacm/src/acm.c     | 2 +-
 librdmacm/rsocket.c | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/ibacm/src/acm.c b/ibacm/src/acm.c
index aba0deaa6de37f..08937345a0c112 100644
--- a/ibacm/src/acm.c
+++ b/ibacm/src/acm.c
@@ -2684,7 +2684,7 @@ static int acmc_init_sa_fds(void)
 		for (p = 0; p < dev->port_cnt; p++) {
 			sa.fds[i].fd = umad_get_fd(dev->port[p].mad_portid);
 			sa.fds[i].events = POLLIN;
-			ret = fcntl(sa.fds[i].fd, F_SETFL, O_NONBLOCK);
+			ret = set_fd_nonblock(sa.fds[i].fd, true);
 			if (ret)
 				acm_log(0, "WARNING - umad fd is blocking\n");
 
diff --git a/librdmacm/rsocket.c b/librdmacm/rsocket.c
index ca0b9bf2a21f7d..693d9eb5cb7e06 100644
--- a/librdmacm/rsocket.c
+++ b/librdmacm/rsocket.c
@@ -49,6 +49,7 @@
 #include <search.h>
 #include <byteswap.h>
 #include <util/compiler.h>
+#include <util/util.h>
 
 #include <rdma/rdma_cma.h>
 #include <rdma/rdma_verbs.h>
@@ -790,7 +791,7 @@ static int rs_create_cq(struct rsocket *rs, struct rdma_cm_id *cm_id)
 		goto err1;
 
 	if (rs->fd_flags & O_NONBLOCK) {
-		if (fcntl(cm_id->recv_cq_channel->fd, F_SETFL, O_NONBLOCK))
+		if (set_fd_nonblock(cm_id->recv_cq_channel->fd, true))
 			goto err2;
 	}
 
@@ -1253,7 +1254,7 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen)
 	}
 
 	if (rs->fd_flags & O_NONBLOCK)
-		fcntl(new_rs->cm_id->channel->fd, F_SETFL, O_NONBLOCK);
+		set_fd_nonblock(new_rs->cm_id->channel->fd, true);
 
 	ret = rs_create_ep(new_rs);
 	if (ret)
@@ -1377,7 +1378,7 @@ connected:
 		break;
 	case rs_accepting:
 		if (!(rs->fd_flags & O_NONBLOCK))
-			fcntl(rs->cm_id->channel->fd, F_SETFL, 0);
+			set_fd_nonblock(rs->cm_id->channel->fd, true);
 
 		ret = ucma_complete(rs->cm_id);
 		if (ret)
-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH rdma-core 4/4] ibacm: Allow systemd to also setup the RDMA_LS socket for ibacm

[Jason Gunthorpe]

This makes sure that very early queries from the kernel can be
delivered to ibacm even before it starts.

Requires trivial systemd patch:
 5570d7f95612 ("Support 'rdma' as a ListenNetlink= argument (#6626)")
Which is present starting in systemd v235

Old systemd's just ignore the netlink listen.

Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
---
 ibacm/ibacm.socket |  4 +++
 ibacm/src/acm.c    | 79 ++++++++++++++++++++++++++++++++++++++++--------------
 2 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/ibacm/ibacm.socket b/ibacm/ibacm.socket
index aa94c91d60daf1..bdf1f9f4e85fe1 100644
--- a/ibacm/ibacm.socket
+++ b/ibacm/ibacm.socket
@@ -10,6 +10,10 @@ Before=rdma-hw.target
 ListenStream=6125
 BindToDevice=lo
 
+# Bind to PF_NETLINK, NETLINK_RDMA, RDMA_NL_GROUP_LS
+# Supported in systemd > 234
+ListenNetlink=rdma 4
+
 [Install]
 # Standard for all sockets
 WantedBy=sockets.target
diff --git a/ibacm/src/acm.c b/ibacm/src/acm.c
index 08937345a0c112..367a43fe9ad1ea 100644
--- a/ibacm/src/acm.c
+++ b/ibacm/src/acm.c
@@ -604,29 +604,56 @@ static int acm_listen(void)
 /* Retrieve the listening socket from systemd. */
 static int acm_listen_systemd(void)
 {
+	int fd;
+
 	int rc = sd_listen_fds(1);
-	if (rc == 0) {
-		/* We are in systemd mode but no FDs were passed? Fall back to
-		 * normal mode
-		 */
-		return acm_listen();
-	}
 	if (rc == -1) {
 		fprintf(stderr, "sd_listen_fds failed %d\n", rc);
 		return rc;
 	}
 
-	if (rc != 1) {
-		fprintf(stderr, "sd_listen_fds returned %d fds, expected 1\n", rc);
+	if (rc > 2) {
+		fprintf(stderr,
+			"sd_listen_fds returned %d fds, expected <= 2\n", rc);
 		return -1;
 	}
 
-	if (!sd_is_socket(SD_LISTEN_FDS_START, AF_UNSPEC, SOCK_STREAM, 1)) {
-		fprintf(stderr, "sd_listen_fds socket is not a SOCK_STREAM listening socket\n");
-		return -1;
+	for (fd = SD_LISTEN_FDS_START; fd != SD_LISTEN_FDS_START + rc; fd++) {
+		if (sd_is_socket(fd, AF_NETLINK, SOCK_RAW, 0)) {
+			/* ListenNetlink for RDMA_NL_GROUP_LS multicast
+			 * messages from the kernel
+			 */
+			if (client_array[NL_CLIENT_INDEX].sock != -1) {
+				fprintf(stderr,
+					"sd_listen_fds returned more than one netlink socket\n");
+				return -1;
+			}
+			client_array[NL_CLIENT_INDEX].sock = fd;
+
+			/* systemd sets NONBLOCK on the netlink socket, while
+			 * we want blocking send to the kernel.
+			 */
+			if (set_fd_nonblock(fd, false)) {
+				fprintf(stderr,
+					"Unable to drop O_NOBLOCK on netlink socket");
+				return -1;
+			}
+		} else if (sd_is_socket(SD_LISTEN_FDS_START, AF_UNSPEC,
+					SOCK_STREAM, 1)) {
+			/* Socket for user space client communication */
+			if (listen_socket != -1) {
+				fprintf(stderr,
+					"sd_listen_fds returned more than one listening socket\n");
+				return -1;
+			}
+			listen_socket = fd;
+		} else {
+			fprintf(stderr,
+				"sd_listen_fds socket is not a SOCK_STREAM/SOCK_NETLINK listening socket\n");
+			return -1;
+		}
 	}
 
-	listen_socket = SD_LISTEN_FDS_START;
 	return 0;
 }
 
@@ -1708,18 +1735,30 @@ static void acm_server(bool systemd)
 
 	acm_log(0, "started\n");
 	acm_init_server();
-	if (systemd)
+
+	client_array[NL_CLIENT_INDEX].sock = -1;
+	listen_socket = -1;
+	if (systemd) {
 		ret = acm_listen_systemd();
-	else
+		if (ret) {
+			acm_log(0, "ERROR - systemd server listen failed\n");
+			return;
+		}
+	}
+
+	if (listen_socket == -1) {
 		ret = acm_listen();
-	if (ret) {
-		acm_log(0, "ERROR - server listen failed\n");
-		return;
+		if (ret) {
+			acm_log(0, "ERROR - server listen failed\n");
+			return;
+		}
 	}
 
-	ret = acm_init_nl();
-	if (ret)
-		acm_log(1, "Warn - Netlink init failed\n");
+	if (client_array[NL_CLIENT_INDEX].sock == -1) {
+		ret = acm_init_nl();
+		if (ret)
+			acm_log(1, "Warn - Netlink init failed\n");
+	}
 
 	if (systemd)
 		sd_notify(0, "READY=1");
-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH rdma-core 3/4] Use set_fd_nonblock in instead of open coding F_SETFL

[Hefty, Sean]

> In all places.
> 
> Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>

Acked-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html