linux-rdma.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Yishai Hadas <yishaih@mellanox.com>
To: linux-rdma@vger.kernel.org
Cc: jgg@mellanox.com, yishaih@mellanox.com, maorg@mellanox.com
Subject: [PATCH rdma-core 04/13] verbs: Handle async FD on an imported device
Date: Wed, 17 Jun 2020 10:45:47 +0300	[thread overview]
Message-ID: <1592379956-7043-5-git-send-email-yishaih@mellanox.com> (raw)
In-Reply-To: <1592379956-7043-1-git-send-email-yishaih@mellanox.com>

An imported device needs to allocate its own async FD to be able to get
events on its private created objects.

This patch use ibv_cmd_alloc_async_fd() to achieve that by forcing any
applicable event's object (e.g QP, SRC, etc.) upon its creation to use
the ioctl interface as now the context async FD becomes a mandatory
attribute.

From symmetric reasons we moved ibv_cmd_alloc_async_fd() to be called as
part of ibv_open_device() in case wasn't allocated yet (i.e. ioctl flow).

Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
---
 libibverbs/cmd_cq.c     |  9 +++++++--
 libibverbs/cmd_device.c |  5 +----
 libibverbs/cmd_qp.c     |  4 ++++
 libibverbs/cmd_srq.c    |  4 ++++
 libibverbs/cmd_wq.c     |  4 ++++
 libibverbs/device.c     | 15 +++++++++++++++
 libibverbs/driver.h     |  1 +
 libibverbs/ibverbs.h    |  1 +
 8 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/libibverbs/cmd_cq.c b/libibverbs/cmd_cq.c
index 1ac5ca5..5412660 100644
--- a/libibverbs/cmd_cq.c
+++ b/libibverbs/cmd_cq.c
@@ -31,6 +31,7 @@
  */
 
 #include <infiniband/cmd_write.h>
+#include "ibverbs.h"
 
 static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
 			      struct ibv_comp_channel *channel, int comp_vector,
@@ -38,6 +39,7 @@ static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
 			      struct ibv_command_buffer *link)
 {
 	DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_CQ, UVERBS_METHOD_CQ_CREATE, 8, link);
+	struct verbs_ex_private *priv = get_priv(context);
 	struct ib_uverbs_attr *handle;
 	struct ib_uverbs_attr *async_fd_attr;
 	uint32_t resp_cqe;
@@ -54,8 +56,11 @@ static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
 		fill_attr_in_fd(cmdb, UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL, channel->fd);
 	fill_attr_in_uint32(cmdb, UVERBS_ATTR_CREATE_CQ_COMP_VECTOR, comp_vector);
 	async_fd_attr = fill_attr_in_fd(cmdb, UVERBS_ATTR_CREATE_CQ_EVENT_FD, context->async_fd);
-	/* Prevent fallback to the 'write' mode if kernel doesn't support it */
-	attr_optional(async_fd_attr);
+	if (priv->imported)
+		fallback_require_ioctl(cmdb);
+	else
+		/* Prevent fallback to the 'write' mode if kernel doesn't support it */
+		attr_optional(async_fd_attr);
 
 	if (flags) {
 		fallback_require_ex(cmdb);
diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c
index 4de59c0..648cc0b 100644
--- a/libibverbs/cmd_device.c
+++ b/libibverbs/cmd_device.c
@@ -99,7 +99,7 @@ int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num,
 	return 0;
 }
 
-static int cmd_alloc_async_fd(struct ibv_context *context)
+int ibv_cmd_alloc_async_fd(struct ibv_context *context)
 {
 	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_ASYNC_EVENT,
 			       UVERBS_METHOD_ASYNC_EVENT_ALLOC, 1);
@@ -153,9 +153,6 @@ static int cmd_get_context(struct verbs_context *context_ex,
 		return 0;
 	}
 	case SUCCESS:
-		ret = cmd_alloc_async_fd(context);
-		if (ret)
-			return ret;
 		break;
 	default:
 		return ret;
diff --git a/libibverbs/cmd_qp.c b/libibverbs/cmd_qp.c
index a11bb98..567984d 100644
--- a/libibverbs/cmd_qp.c
+++ b/libibverbs/cmd_qp.c
@@ -76,6 +76,7 @@ static int ibv_icmd_create_qp(struct ibv_context *context,
 			      struct ibv_command_buffer *link)
 {
 	DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_QP, UVERBS_METHOD_QP_CREATE, 15, link);
+	struct verbs_ex_private *priv = get_priv(context);
 	struct ib_uverbs_attr *handle;
 	uint32_t qp_num;
 	uint32_t pd_handle;
@@ -172,6 +173,9 @@ static int ibv_icmd_create_qp(struct ibv_context *context,
 	fill_attr_in_ptr(cmdb, UVERBS_ATTR_CREATE_QP_CAP, &attr_ex->cap);
 	fill_attr_in_fd(cmdb, UVERBS_ATTR_CREATE_QP_EVENT_FD, context->async_fd);
 
+	if (priv->imported)
+		fallback_require_ioctl(cmdb);
+
 	if (attr_ex->sq_sig_all)
 		create_flags |= IB_UVERBS_QP_CREATE_SQ_SIG_ALL;
 
diff --git a/libibverbs/cmd_srq.c b/libibverbs/cmd_srq.c
index 4e63046..dfaaa6a 100644
--- a/libibverbs/cmd_srq.c
+++ b/libibverbs/cmd_srq.c
@@ -53,6 +53,7 @@ static int ibv_icmd_create_srq(struct ibv_pd *pd, struct verbs_srq *vsrq,
 			       struct ibv_command_buffer *link)
 {
 	DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_SRQ, UVERBS_METHOD_SRQ_CREATE, 13, link);
+	struct verbs_ex_private *priv = get_priv(pd->context);
 	struct ib_uverbs_attr *handle;
 	uint32_t max_wr;
 	uint32_t max_sge;
@@ -107,6 +108,9 @@ static int ibv_icmd_create_srq(struct ibv_pd *pd, struct verbs_srq *vsrq,
 	fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_SRQ_RESP_MAX_WR, &max_wr);
 	fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_SRQ_RESP_MAX_SGE, &max_sge);
 
+	if (priv->imported)
+		fallback_require_ioctl(cmdb);
+
 	switch (execute_ioctl_fallback(srq->context, create_srq, cmdb, &ret)) {
 	case TRY_WRITE: {
 		if (attr_ex->srq_type == IBV_SRQT_BASIC && abi_ver > 5) {
diff --git a/libibverbs/cmd_wq.c b/libibverbs/cmd_wq.c
index d233c3a..855fc04 100644
--- a/libibverbs/cmd_wq.c
+++ b/libibverbs/cmd_wq.c
@@ -31,6 +31,7 @@
  */
 
 #include <infiniband/cmd_write.h>
+#include "ibverbs.h"
 
 static int ibv_icmd_create_wq(struct ibv_context *context,
 			      struct ibv_wq_init_attr *wq_init_attr,
@@ -38,6 +39,7 @@ static int ibv_icmd_create_wq(struct ibv_context *context,
 			      struct ibv_command_buffer *link)
 {
 	DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_WQ, UVERBS_METHOD_WQ_CREATE, 13, link);
+	struct verbs_ex_private *priv = get_priv(context);
 	struct ib_uverbs_attr *handle;
 	uint32_t max_wr;
 	uint32_t max_sge;
@@ -62,6 +64,8 @@ static int ibv_icmd_create_wq(struct ibv_context *context,
 	fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_WQ_RESP_MAX_SGE, &max_sge);
 	fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_WQ_RESP_WQ_NUM, &wq_num);
 
+	if (priv->imported)
+		fallback_require_ioctl(cmdb);
 	fallback_require_ex(cmdb);
 
 	switch (execute_ioctl_fallback(context, create_wq, cmdb, &ret)) {
diff --git a/libibverbs/device.c b/libibverbs/device.c
index 0a8403d..595af82 100644
--- a/libibverbs/device.c
+++ b/libibverbs/device.c
@@ -344,6 +344,7 @@ struct ibv_context *verbs_open_device(struct ibv_device *device, void *private_d
 	struct verbs_device *verbs_device = verbs_get_device(device);
 	int cmd_fd;
 	struct verbs_context *context_ex;
+	int ret;
 
 	/*
 	 * We'll only be doing writes, but we need O_RDWR in case the
@@ -363,6 +364,13 @@ struct ibv_context *verbs_open_device(struct ibv_device *device, void *private_d
 		return NULL;
 
 	set_lib_ops(context_ex);
+	if (context_ex->context.async_fd == -1) {
+		ret = ibv_cmd_alloc_async_fd(&context_ex->context);
+		if (ret) {
+			ibv_close_device(&context_ex->context);
+			return NULL;
+		}
+	}
 
 	return &context_ex->context;
 }
@@ -381,6 +389,7 @@ static struct ibv_context *verbs_import_device(int cmd_fd)
 	struct ibv_device **dev_list;
 	struct ibv_context *ctx = NULL;
 	struct stat st;
+	int ret;
 	int i;
 
 	if (fstat(cmd_fd, &st) || !S_ISCHR(st.st_mode)) {
@@ -418,7 +427,13 @@ static struct ibv_context *verbs_import_device(int cmd_fd)
 
 	set_lib_ops(context_ex);
 
+	context_ex->priv->imported = true;
 	ctx = &context_ex->context;
+	ret = ibv_cmd_alloc_async_fd(ctx);
+	if (ret) {
+		ibv_close_device(ctx);
+		ctx = NULL;
+	}
 out:
 	ibv_free_device_list(dev_list);
 	return ctx;
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index 52ecbfe..98a1b24 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -446,6 +446,7 @@ int ibv_cmd_query_device_ex(struct ibv_context *context,
 int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num,
 		       struct ibv_port_attr *port_attr,
 		       struct ibv_query_port *cmd, size_t cmd_size);
+int ibv_cmd_alloc_async_fd(struct ibv_context *context);
 int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
 		     struct ibv_alloc_pd *cmd, size_t cmd_size,
 		     struct ib_uverbs_alloc_pd_resp *resp, size_t resp_size);
diff --git a/libibverbs/ibverbs.h b/libibverbs/ibverbs.h
index 4b9b88f..6703c10 100644
--- a/libibverbs/ibverbs.h
+++ b/libibverbs/ibverbs.h
@@ -74,6 +74,7 @@ struct verbs_ex_private {
 	uint32_t driver_id;
 	bool use_ioctl_write;
 	struct verbs_context_ops ops;
+	bool imported;
 };
 
 static inline struct verbs_ex_private *get_priv(struct ibv_context *ctx)
-- 
1.8.3.1


  parent reply	other threads:[~2020-06-17  7:46 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-06-17  7:45 [PATCH rdma-core 00/13] verbs: Introduce import verbs for device, PD, MR Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 01/13] Update kernel headers Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 02/13] verbs: Close async_fd only when it was previously created Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 03/13] verbs: Introduce ibv_import_device() verb Yishai Hadas
2020-06-19 12:29   ` Jason Gunthorpe
2020-06-21  7:01     ` Yishai Hadas
2020-06-22 12:52       ` Jason Gunthorpe
2020-06-23 13:06         ` Yishai Hadas
2020-06-17  7:45 ` Yishai Hadas [this message]
2020-06-19 12:33   ` [PATCH rdma-core 04/13] verbs: Handle async FD on an imported device Jason Gunthorpe
2020-06-21  9:08     ` Yishai Hadas
2020-06-23 17:34       ` Jason Gunthorpe
2020-06-24  7:22         ` Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 05/13] mlx5: Refactor mlx5_alloc_context() Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 06/13] mlx5: Implement the import device functionality Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 07/13] verbs: Introduce ibv_import/unimport_pd() verbs Yishai Hadas
2020-06-19 12:48   ` Jason Gunthorpe
2020-06-21  8:30     ` Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 08/13] mlx5: Implement the import/unimport PD verbs Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 09/13] verbs: Introduce ibv_import/unimport_mr() verbs Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 10/13] mlx5: Implement the import/unimport MR verbs Yishai Hadas
2020-06-19 12:50   ` Jason Gunthorpe
2020-06-21  8:44     ` Yishai Hadas
2020-06-23 17:33       ` Jason Gunthorpe
2020-06-24  7:20         ` Yishai Hadas
2020-07-01 12:28         ` Yishai Hadas
2020-07-02 17:42           ` Jason Gunthorpe
2020-06-17  7:45 ` [PATCH rdma-core 11/13] pyverbs: Support verbs import APIs Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 12/13] Documentation: Add usage example for verbs import Yishai Hadas
2020-06-17  7:45 ` [PATCH rdma-core 13/13] tests: Add a shared PD Pyverbs test Yishai Hadas

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1592379956-7043-5-git-send-email-yishaih@mellanox.com \
    --to=yishaih@mellanox.com \
    --cc=jgg@mellanox.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=maorg@mellanox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).